From e4d81294a9153fb099713c06c720676af87f6b62 Mon Sep 17 00:00:00 2001 From: Shruti Mantri Date: Fri, 15 Nov 2024 20:29:36 +0530 Subject: [PATCH] fix: remove reference of EachSequential and EachParallel --- azure-blob-to-bigquery.yaml | 5 +++-- parallel-files.yaml | 5 +++-- parallel-python.yaml | 5 +++-- python-csv-each-parallel.yaml | 5 +++-- python-partitions-metrics.yaml | 5 +++-- redis-set-parallel.yaml | 5 +++-- s3-map-over-objects.yaml | 5 +++-- s3-trigger.yaml | 7 ++++--- snowflake-query-trigger.yaml | 4 ++-- 9 files changed, 27 insertions(+), 19 deletions(-) diff --git a/azure-blob-to-bigquery.yaml b/azure-blob-to-bigquery.yaml index 176b1ac..9132265 100644 --- a/azure-blob-to-bigquery.yaml +++ b/azure-blob-to-bigquery.yaml @@ -3,8 +3,9 @@ namespace: company.team tasks: - id: each - type: io.kestra.plugin.core.flow.EachParallel - value: "{{ trigger.blobs | jq('.[].uri') }}" + type: io.kestra.plugin.core.flow.ForEach + concurrencyLimit: 0 + values: "{{ trigger.blobs | jq('.[].uri') }}" tasks: - id: upload_from_file type: io.kestra.plugin.gcp.bigquery.Load diff --git a/parallel-files.yaml b/parallel-files.yaml index 1e0a727..9452f20 100644 --- a/parallel-files.yaml +++ b/parallel-files.yaml @@ -16,8 +16,9 @@ tasks: - echo "Hello from 4" >> out/output4.txt - id: each - type: io.kestra.plugin.core.flow.EachParallel - value: "{{ outputs.bash.outputFiles | jq('.[]') }}" + type: io.kestra.plugin.core.flow.ForEach + concurrencyLimit: 0 + values: "{{ outputs.bash.outputFiles | jq('.[]') }}" tasks: - id: path type: io.kestra.plugin.core.debug.Return diff --git a/parallel-python.yaml b/parallel-python.yaml index e31104e..e475b5f 100644 --- a/parallel-python.yaml +++ b/parallel-python.yaml @@ -3,8 +3,9 @@ namespace: company.team tasks: - id: parallel - type: io.kestra.plugin.core.flow.EachParallel - value: + type: io.kestra.plugin.core.flow.ForEach + concurrencyLimit: 0 + values: - 1 - 2 - 3 diff --git a/python-csv-each-parallel.yaml b/python-csv-each-parallel.yaml index 24243e7..9f29627 100644 --- a/python-csv-each-parallel.yaml +++ b/python-csv-each-parallel.yaml @@ -3,8 +3,9 @@ namespace: company.team tasks: - id: csv - type: io.kestra.plugin.core.flow.EachParallel - value: + type: io.kestra.plugin.core.flow.ForEach + concurrencyLimit: 0 + values: - https://huggingface.co/datasets/kestra/datasets/raw/main/csv/orders.csv - https://huggingface.co/datasets/kestra/datasets/raw/main/csv/products.csv - https://huggingface.co/datasets/kestra/datasets/raw/main/csv/salaries.csv diff --git a/python-partitions-metrics.yaml b/python-partitions-metrics.yaml index e874ccb..294d578 100644 --- a/python-partitions-metrics.yaml +++ b/python-partitions-metrics.yaml @@ -14,8 +14,9 @@ tasks: Kestra.outputs({'partitions': partitions}) - id: process_partitions - type: io.kestra.plugin.core.flow.EachParallel - value: "{{ outputs.get_partitions.vars.partitions }}" + type: io.kestra.plugin.core.flow.ForEach + concurrencyLimit: 0 + values: "{{ outputs.get_partitions.vars.partitions }}" tasks: - id: partition type: io.kestra.plugin.scripts.python.Script diff --git a/redis-set-parallel.yaml b/redis-set-parallel.yaml index e69a0e3..9da48ef 100644 --- a/redis-set-parallel.yaml +++ b/redis-set-parallel.yaml @@ -14,8 +14,9 @@ inputs: tasks: - id: parallel - type: io.kestra.plugin.core.flow.EachParallel - value: "{{ inputs.values }}" + type: io.kestra.plugin.core.flow.ForEach + concurrencyLimit: 0 + values: "{{ inputs.values }}" tasks: - id: set type: io.kestra.plugin.redis.string.Set diff --git a/s3-map-over-objects.yaml b/s3-map-over-objects.yaml index 5530e66..cd1a600 100644 --- a/s3-map-over-objects.yaml +++ b/s3-map-over-objects.yaml @@ -20,8 +20,9 @@ tasks: message: "Found objects {{ outputs.list_objects.objects }}" - id: map_over_s3_objects - type: io.kestra.plugin.core.flow.EachParallel - value: "{{ outputs.list_objects.objects }}" + type: io.kestra.plugin.core.flow.ForEach + concurrencyLimit: 0 + values: "{{ outputs.list_objects.objects }}" tasks: - id: filename type: io.kestra.plugin.core.log.Log diff --git a/s3-trigger.yaml b/s3-trigger.yaml index 465056f..cdcb4df 100644 --- a/s3-trigger.yaml +++ b/s3-trigger.yaml @@ -3,12 +3,13 @@ namespace: company.team tasks: - id: each - type: io.kestra.plugin.core.flow.EachParallel + type: io.kestra.plugin.core.flow.ForEach + concurrencyLimit: 0 + values: "{{ trigger.objects | jq('.[].uri') }}" tasks: - id: s3_object type: io.kestra.plugin.core.debug.Return format: "{{ taskrun.value }}" - value: "{{ trigger.objects | jq('.[].uri') }}" triggers: - id: wait_for_s3_object @@ -32,7 +33,7 @@ extend: internal storage and move the S3 objects to an `archive` folder (i.e. S3 object prefix with the name `archive`). - The `EachParallel` task will iterate over the objects and print their URIs. + The `FoEach` task will iterate over the objects and print their URIs. It's recommended to set the `accessKeyId` and `secretKeyId` properties as secrets. diff --git a/snowflake-query-trigger.yaml b/snowflake-query-trigger.yaml index b94c9c1..1ac2a15 100644 --- a/snowflake-query-trigger.yaml +++ b/snowflake-query-trigger.yaml @@ -3,8 +3,8 @@ namespace: company.team tasks: - id: each - type: io.kestra.plugin.core.flow.EachSequential - value: "{{ trigger.rows }}" + type: io.kestra.plugin.core.flow.ForEach + values: "{{ trigger.rows }}" tasks: - id: automated_process type: io.kestra.plugin.scripts.shell.Commands