From b694e5f69f1d178ff821c51630ec44f46ff1d053 Mon Sep 17 00:00:00 2001 From: Alisson Marcelo Date: Thu, 17 Aug 2023 14:08:02 -0300 Subject: [PATCH 1/2] refactor: improving data streaming openapi to prevent health check errors --- data_streaming.yml | 100 +++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 97 insertions(+), 3 deletions(-) diff --git a/data_streaming.yml b/data_streaming.yml index bc064d0a..a809529d 100644 --- a/data_streaming.yml +++ b/data_streaming.yml @@ -464,7 +464,14 @@ components: Note: * `Range` - From 0 to 100. * `To use:` [Contact the sales team](https://www.azion.com/en/contact-sales/) to activate this feature in your account. - + + required: + - name + - template_id + - endpoint + - domains_ids + + StandardDataStreamingPostBody: type: object properties: @@ -503,7 +510,21 @@ components: default: true endpoint: - $ref: '#/components/schemas/DataStreamingEndpointTypeStandard' + type: string + format: json + description: > + Options' examples: + - `Standard HTTP/HTTPS POST` - { "endpoint_type": "standard", "url": "http://example.com", "log_line_separator": "\n", "payload_format": "$dataset", "max_size": 1000024 } + - `Apache Kafka` - { "endpoint_type": "kafka", "kafka_topic": "example_topic", "bootstrap_servers": "kafka-server.com:9092,kafka-server-2.com:9092", "use_tls":true } + - `Simple Storage Service (S3)` - { "endpoint_type": "s3", "access_key": "MYACCESSKEY", "region": "us-east-1", "object_key_prefix": "my_prefix_", "bucket_name": "bucket_example", "content_type": "plain/text", "host_url": "http://aws-host.com", "secret_key": "MYSECRETKEY" } + - `Google BigQuery` - { "endpoint_type": "big_query", "dataset_id": "my_dataset", "project_id": "my_project", "table_id": "my_table", "service_account_key": "{ "service_account_key": "key_content" }" } + - `Elasticsearch` - { “endpoint_type”: "elasticsearch", “url”: “http://elasticsearch.com”, “api_key”: “XYZ_API_KEY” } + - `AWS Kinesis Data Firehose` - { "endpoint_type": "aws_kinesis_firehose", "access_key": "MYACCESSKEY", "stream_name": "my_stream_name", "region": "us-east-1", "secret_key": "MYSECRETKEY" } + - `Datadog` - { "endpoint_type": "datadog", "url": "https://http-intake.logs.datadoghq.com/v1/input", "api_key": "MYAPIKEY" } + - `IBM QRadar` - { "endpoint_type": "qradar", "url": "http://137.15.824.10:14440” } + - `Azure Monitor` - { "endpoint_type": "azure_monitor", "log_type": "myLogType", "shared_key": "mysharedkey", "time_generated_field": "timeGeneratedField", "workspace_id": "anfhw-123sd-466gcs"} + - `Azure Blob Storage` - { "endpoint_type": "azure_blob_storage", "storage_account": "mystorageaccount", "container_name": "log_container", "blob_sas_token": "fd56e23e1f12efe" } + - `Splunk` - { "endpoint_type": "splunk", "url": "https://inputs.splunk-client.splunkcloud.com:1337/services/collector", "api_key": "MYAPIKEY" } domains_ids: type: array @@ -513,6 +534,8 @@ components: description: > Note: * Field not used with the rtm_activity data source. + # OpenAPI 3.0 does not support parameter dependencies and mutually exclusive parameters. + # Link: https://stackoverflow.com/questions/63209596/how-to-indicate-that-a-parameter-is-conditionally-required-when-another-paramete/63213464#63213464 all_domains: type: boolean @@ -521,6 +544,24 @@ components: description: > Note: * Field not used with the rtm_activity data source. + # OpenAPI 3.0 does not support parameter dependencies and mutually exclusive parameters. + # Link: https://stackoverflow.com/questions/63209596/how-to-indicate-that-a-parameter-is-conditionally-required-when-another-paramete/63213464#63213464 + + sampling_percentage: + type: integer + nullable: true + minimum: 0 + maximum: 100 + description: > + Note: + * `Range` - From 0 to 100. + * `To use:` [Contact the sales team](https://www.azion.com/en/contact-sales/) to activate this feature in your account. + + required: + - name + - template_id + - endpoint + - domains_ids CustomDataStreamingPostBody: type: object @@ -551,6 +592,59 @@ components: nullable: true default: true + endpoint: + type: string + format: json + description: > + Options' examples: + - `Standard HTTP/HTTPS POST` - { "endpoint_type": "standard", "url": "http://example.com", "log_line_separator": "\n", "payload_format": "$dataset", "max_size": 1000024 } + - `Apache Kafka` - { "endpoint_type": "kafka", "kafka_topic": "example_topic", "bootstrap_servers": "kafka-server.com:9092,kafka-server-2.com:9092", "use_tls":true } + - `Simple Storage Service (S3)` - { "endpoint_type": "s3", "access_key": "MYACCESSKEY", "region": "us-east-1", "object_key_prefix": "my_prefix_", "bucket_name": "bucket_example", "content_type": "plain/text", "host_url": "http://aws-host.com", "secret_key": "MYSECRETKEY" } + - `Google BigQuery` - { "endpoint_type": "big_query", "dataset_id": "my_dataset", "project_id": "my_project", "table_id": "my_table", "service_account_key": "{ "service_account_key": "key_content" }" } + - `Elasticsearch` - { “endpoint_type”: "elasticsearch", “url”: “http://elasticsearch.com”, “api_key”: “XYZ_API_KEY” } + - `AWS Kinesis Data Firehose` - { "endpoint_type": "aws_kinesis_firehose", "access_key": "MYACCESSKEY", "stream_name": "my_stream_name", "region": "us-east-1", "secret_key": "MYSECRETKEY" } + - `Datadog` - { "endpoint_type": "datadog", "url": "https://http-intake.logs.datadoghq.com/v1/input", "api_key": "MYAPIKEY" } + - `IBM QRadar` - { "endpoint_type": "qradar", "url": "http://137.15.824.10:14440” } + - `Azure Monitor` - { "endpoint_type": "azure_monitor", "log_type": "myLogType", "shared_key": "mysharedkey", "time_generated_field": "timeGeneratedField", "workspace_id": "anfhw-123sd-466gcs"} + - `Azure Blob Storage` - { "endpoint_type": "azure_blob_storage", "storage_account": "mystorageaccount", "container_name": "log_container", "blob_sas_token": "fd56e23e1f12efe" } + - `Splunk` - { "endpoint_type": "splunk", "url": "https://inputs.splunk-client.splunkcloud.com:1337/services/collector", "api_key": "MYAPIKEY" } + + domains_ids: + type: array + items: + type: integer + minItems: 1 + description: > + Note: + * Field not used with the rtm_activity data source. + # OpenAPI 3.0 does not support parameter dependencies and mutually exclusive parameters. + # Link: https://stackoverflow.com/questions/63209596/how-to-indicate-that-a-parameter-is-conditionally-required-when-another-paramete/63213464#63213464 + + all_domains: + type: boolean + nullable: true + default: false + description: > + Note: + * Field not used with the rtm_activity data source. + # OpenAPI 3.0 does not support parameter dependencies and mutually exclusive parameters. + # Link: https://stackoverflow.com/questions/63209596/how-to-indicate-that-a-parameter-is-conditionally-required-when-another-paramete/63213464#63213464 + + sampling_percentage: + type: integer + nullable: true + minimum: 0 + maximum: 100 + description: > + Note: + * `Range` - From 0 to 100. + * `To use:` [Contact the sales team](https://www.azion.com/en/contact-sales/) to activate this feature in your account. + + required: + - name + - template_id + - endpoint + - domains_ids CreateDataStreamingResponse: @@ -1072,4 +1166,4 @@ components: description: | You must inform a token to auth. Usage format: `Token ` - \ No newline at end of file + From a417e66cfd7f24f9e3edf3b5d7a8b13c5d971ce5 Mon Sep 17 00:00:00 2001 From: Alisson Marcelo Date: Thu, 17 Aug 2023 15:26:16 -0300 Subject: [PATCH 2/2] refactor: using endpoint definitions instead of string --- data_streaming.yml | 122 +++++++++++++++++++++++++++++++++++++++------ 1 file changed, 107 insertions(+), 15 deletions(-) diff --git a/data_streaming.yml b/data_streaming.yml index a809529d..e1d85cc9 100644 --- a/data_streaming.yml +++ b/data_streaming.yml @@ -418,8 +418,19 @@ components: default: true endpoint: - type: string - format: json + oneOf: + - $ref: '#/components/schemas/Endpoint_Standard' + - $ref: '#/components/schemas/Endpoint_Kafka' + - $ref: '#/components/schemas/Endpoint_S3' + - $ref: '#/components/schemas/Endpoint_Google_BigQuery' + - $ref: '#/components/schemas/Endpoint_Elasticsearch' + - $ref: '#/components/schemas/Endpoint_AWS_Kinesis_Firehose' + - $ref: '#/components/schemas/Endpoint_Datadog' + - $ref: '#/components/schemas/Endpoint_IBM_QRadar' + - $ref: '#/components/schemas/Endpoint_Azure_Monitor' + - $ref: '#/components/schemas/Endpoint_Azure_Blob_Storage' + - $ref: '#/components/schemas/Endpoint_Splunk' + description: > Options' examples: - `Standard HTTP/HTTPS POST` - { "endpoint_type": "standard", "url": "http://example.com", "log_line_separator": "\n", "payload_format": "$dataset", "max_size": 1000024 } @@ -510,8 +521,18 @@ components: default: true endpoint: - type: string - format: json + oneOf: + - $ref: '#/components/schemas/Endpoint_Standard' + - $ref: '#/components/schemas/Endpoint_Kafka' + - $ref: '#/components/schemas/Endpoint_S3' + - $ref: '#/components/schemas/Endpoint_Google_BigQuery' + - $ref: '#/components/schemas/Endpoint_Elasticsearch' + - $ref: '#/components/schemas/Endpoint_AWS_Kinesis_Firehose' + - $ref: '#/components/schemas/Endpoint_Datadog' + - $ref: '#/components/schemas/Endpoint_IBM_QRadar' + - $ref: '#/components/schemas/Endpoint_Azure_Monitor' + - $ref: '#/components/schemas/Endpoint_Azure_Blob_Storage' + - $ref: '#/components/schemas/Endpoint_Splunk' description: > Options' examples: - `Standard HTTP/HTTPS POST` - { "endpoint_type": "standard", "url": "http://example.com", "log_line_separator": "\n", "payload_format": "$dataset", "max_size": 1000024 } @@ -593,8 +614,18 @@ components: default: true endpoint: - type: string - format: json + oneOf: + - $ref: '#/components/schemas/Endpoint_Standard' + - $ref: '#/components/schemas/Endpoint_Kafka' + - $ref: '#/components/schemas/Endpoint_S3' + - $ref: '#/components/schemas/Endpoint_Google_BigQuery' + - $ref: '#/components/schemas/Endpoint_Elasticsearch' + - $ref: '#/components/schemas/Endpoint_AWS_Kinesis_Firehose' + - $ref: '#/components/schemas/Endpoint_Datadog' + - $ref: '#/components/schemas/Endpoint_IBM_QRadar' + - $ref: '#/components/schemas/Endpoint_Azure_Monitor' + - $ref: '#/components/schemas/Endpoint_Azure_Blob_Storage' + - $ref: '#/components/schemas/Endpoint_Splunk' description: > Options' examples: - `Standard HTTP/HTTPS POST` - { "endpoint_type": "standard", "url": "http://example.com", "log_line_separator": "\n", "payload_format": "$dataset", "max_size": 1000024 } @@ -710,9 +741,9 @@ components: type: array items: oneOf: - - $ref: '#/components/schemas/Endpoint_Default' + - $ref: '#/components/schemas/Endpoint_Standard' - $ref: '#/components/schemas/Endpoint_Kafka' - - $ref: '#/components/schemas/Endpoinrt_S3' + - $ref: '#/components/schemas/Endpoint_S3' - $ref: '#/components/schemas/Endpoint_Google_BigQuery' - $ref: '#/components/schemas/Endpoint_Elasticsearch' - $ref: '#/components/schemas/Endpoint_AWS_Kinesis_Firehose' @@ -773,7 +804,7 @@ components: Note: * Field not used with the rtm_activity data source. - Endpoint_Default: + Endpoint_Standard: type: object properties: endpoint_type: @@ -797,7 +828,11 @@ components: example: additional_header: additional_header_value maximum: 5 - + required: + - endpoint_type + - url + - payload_format + - log_line_separator Endpoint_Kafka: type: object @@ -811,8 +846,12 @@ components: type: string use_tls: type: boolean - - Endpoinrt_S3: + required: + - endpoint_type + - kafka_topic + - bootstrap_servers + + Endpoint_S3: type: object properties: endpoint_type: @@ -833,6 +872,14 @@ components: format: uri secret_key: type: string + required: + - endpoint_type + - access_key + - secret_key + - content_type + - bucket_name + - host_url + - region Endpoint_Google_BigQuery: type: object @@ -851,6 +898,12 @@ components: properties: service_account_key: type: string + required: + - endpoint_type + - project_id + - table_id + - service_account_key + - dataset_id Endpoint_Elasticsearch: type: object @@ -863,6 +916,10 @@ components: format: uri api_key: format: string + required: + - endpoint_type + - url + - api_key Endpoint_AWS_Kinesis_Firehose: type: object @@ -878,18 +935,28 @@ components: type: string secret_key: type: string - + required: + - endpoint_type + - access_key + - secret_key + - stream_name + - region + Endpoint_Datadog: type: object properties: endpoint_type: type: string example: "datadog" - url: + datadog_url: type: string format: uri api_key: format: string + required: + - endpoint_type + - api_key + - datadog_url Endpoint_IBM_QRadar: type: object @@ -900,6 +967,9 @@ components: url: type: string format: uri + required: + - endpoint_type + - url Endpoint_Azure_Monitor: type: object @@ -916,6 +986,11 @@ components: workspace_id: type: string pattern: '[a-zA-Z0-9]{5}-[a-zA-Z0-9]{5}-[a-zA-Z0-9]{6}' + required: + - endpoint_type + - log_type + - shared_key + - workspace_id Endpoint_Azure_Blob_Storage: type: object @@ -930,6 +1005,11 @@ components: blob_sas_token: type: string pattern: '[a-zA-Z0-9]{15}' + required: + - endpoint_type + - storage_account + - container_name + - blob_sas_token Endpoint_Splunk: type: object @@ -942,6 +1022,10 @@ components: format: uri api_key: type: string + required: + - endpoint_type + - api_key + - url DataStreamingResponseWithResults: @@ -1004,6 +1088,7 @@ components: $ref: '#/components/schemas/DataStreamingEndpointTypeKafka' all_domains: type: boolean + DataStreamingEndpointTypeKafka: type: object @@ -1016,7 +1101,10 @@ components: type: string bootstrap_servers: type: string - + required: + - kafka_topic + - bootstrap_servers + DataStreamingResponseGetResultTypeStandard: type: object properties: @@ -1051,6 +1139,10 @@ components: type: integer headers: $ref: '#/components/schemas/DataStreamingEndpointTypeStandardHeadersExample' + required: + - url + - payload_format + - log_line_separator DataStreamingEndpointTypeStandardHeadersExample: type: object