feat: Expand environment variables in array setting values (meltano#8268

) * fix: Expand enviroment variables in array setting values Closes meltano#3171 * Do not support `flat=True` for expanding env vars in lists * Test with flat array * Remove redundant test * Update src/meltano/core/utils/__init__.py Co-authored-by: @WillDaSilva --------- Co-authored-by: Will Da Silva <[email protected]>
jaceksan · Dec 4, 2023 · f2358d3 · f2358d3
1 parent 4fa64ab
commit f2358d3
Show file tree

Hide file tree

Showing 12 changed files with 184 additions and 3 deletions.
diff --git a/.github/workflows/integration_tests.yml b/.github/workflows/integration_tests.yml
@@ -39,6 +39,7 @@ jobs:
         - { integration_test: "meltano-config", needs_postgres: false}
         - { integration_test: "meltano-annotations", needs_postgres: false}
         - { integration_test: "meltano-manifest", needs_postgres: false}
+        - { integration_test: "meltano-expand-envvars-in-array", needs_postgres: false}
       fail-fast: false
 
     runs-on: ubuntu-latest

diff --git a/integration/example-library/meltano-expand-envvars-in-array/.gitignore b/integration/example-library/meltano-expand-envvars-in-array/.gitignore
@@ -0,0 +1,3 @@
+/venv
+/.meltano
+.env
diff --git a/integration/example-library/meltano-expand-envvars-in-array/data/.gitignore b/integration/example-library/meltano-expand-envvars-in-array/data/.gitignore
@@ -0,0 +1 @@
+!*.jsonl
diff --git a/integration/example-library/meltano-expand-envvars-in-array/data/birds.jsonl b/integration/example-library/meltano-expand-envvars-in-array/data/birds.jsonl
@@ -0,0 +1,10 @@
+{"id": 1, "description": "Bald Eagle", "verified": true, "views": 89, "created_at": "2022-03-15T09:45:30Z"}
+{"id": 2, "description": "American Robin", "verified": true, "views": 72, "created_at": "2022-05-18T14:20:12Z"}
+{"id": 3, "description": "Peregrine Falcon", "verified": true, "views": 65, "created_at": "2022-07-10T17:55:42Z"}
+{"id": 4, "description": "Northern Cardinal", "verified": true, "views": 51, "created_at": "2022-09-04T21:12:58Z"}
+{"id": 5, "description": "Mallard Duck", "verified": true, "views": 47, "created_at": "2022-11-09T11:30:25Z"}
+{"id": 6, "description": "European Swallow", "verified": true, "views": 39, "created_at": "2023-01-22T08:15:37Z"}
+{"id": 7, "description": "Ruby-throated Hummingbird", "verified": true, "views": 54, "created_at": "2023-03-05T14:48:19Z"}
+{"id": 8, "description": "Great Horned Owl", "verified": true, "views": 68, "created_at": "2023-05-14T20:05:52Z"}
+{"id": 9, "description": "European Starling", "verified": true, "views": 42, "created_at": "2023-07-19T09:10:36Z"}
+{"id": 10, "description": "Blue Jay", "verified": true, "views": 59, "created_at": "2023-09-30T16:27:04Z"}
diff --git a/integration/example-library/meltano-expand-envvars-in-array/data/mammals.jsonl b/integration/example-library/meltano-expand-envvars-in-array/data/mammals.jsonl
@@ -0,0 +1,10 @@
+{"id": 1, "description": "African Elephant", "verified": true, "views": 125, "created_at": "2022-03-15T09:45:30Z"}
+{"id": 2, "description": "Bengal Tiger", "verified": true, "views": 98, "created_at": "2022-05-18T14:20:12Z"}
+{"id": 3, "description": "Koala", "verified": true, "views": 73, "created_at": "2022-07-10T17:55:42Z"}
+{"id": 4, "description": "Gray Wolf", "verified": true, "views": 55, "created_at": "2022-09-04T21:12:58Z"}
+{"id": 5, "description": "Giraffe", "verified": true, "views": 62, "created_at": "2022-11-09T11:30:25Z"}
+{"id": 6, "description": "Panda Bear", "verified": true, "views": 48, "created_at": "2023-01-22T08:15:37Z"}
+{"id": 7, "description": "Dolphin", "verified": true, "views": 77, "created_at": "2023-03-05T14:48:19Z"}
+{"id": 8, "description": "Red Fox", "verified": true, "views": 36, "created_at": "2023-05-14T20:05:52Z"}
+{"id": 9, "description": "Orangutan", "verified": true, "views": 43, "created_at": "2023-07-19T09:10:36Z"}
+{"id": 10, "description": "Grizzly Bear", "verified": true, "views": 50, "created_at": "2023-09-30T16:27:04Z"}
diff --git a/integration/example-library/meltano-expand-envvars-in-array/data/reptiles.jsonl b/integration/example-library/meltano-expand-envvars-in-array/data/reptiles.jsonl
@@ -0,0 +1,10 @@
+{"id": 1, "description": "Komodo Dragon", "verified": true, "views": 74, "created_at": "2022-03-15T09:45:30Z"}
+{"id": 2, "description": "Green Sea Turtle", "verified": true, "views": 63, "created_at": "2022-05-18T14:20:12Z"}
+{"id": 3, "description": "King Cobra", "verified": true, "views": 58, "created_at": "2022-07-10T17:55:42Z"}
+{"id": 4, "description": "Gila Monster", "verified": true, "views": 42, "created_at": "2022-09-04T21:12:58Z"}
+{"id": 5, "description": "Galapagos Tortoise", "verified": true, "views": 53, "created_at": "2022-11-09T11:30:25Z"}
+{"id": 6, "description": "American Alligator", "verified": true, "views": 67, "created_at": "2023-01-22T08:15:37Z"}
+{"id": 7, "description": "Western Diamondback Rattlesnake", "verified": true, "views": 49, "created_at": "2023-03-05T14:48:19Z"}
+{"id": 8, "description": "Chameleon", "verified": true, "views": 38, "created_at": "2023-05-14T20:05:52Z"}
+{"id": 9, "description": "Iguana", "verified": true, "views": 45, "created_at": "2023-07-19T09:10:36Z"}
+{"id": 10, "description": "Leopard Gecko", "verified": true, "views": 55, "created_at": "2023-09-30T16:27:04Z"}
diff --git a/integration/example-library/meltano-expand-envvars-in-array/ending-meltano.yml b/integration/example-library/meltano-expand-envvars-in-array/ending-meltano.yml
@@ -0,0 +1,27 @@
+version: 1
+default_environment: dev
+project_id: 95ed0825-997f-4baf-9c13-bae04ac44fe1
+environments:
+- name: dev
+- name: staging
+- name: prod
+send_anonymous_usage_stats: false
+plugins:
+  extractors:
+  - name: tap-smoke-test
+    namespace: tap_smoke_test
+    pip_url: git+https://github.com/meltano/tap-smoke-test.git
+    executable: tap-smoke-test
+    capabilities:
+    - discover
+    - catalog
+    settings:
+    - name: schema_inference_record_count
+      kind: integer
+    - name: streams
+      kind: array
+    config:
+      schema_inference_record_count: 5
+      streams:
+      - stream_name: $ANIMAL_GROUP
+        input_filename: $MELTANO_PROJECT_ROOT/data/$ANIMAL_GROUP.jsonl
diff --git a/integration/example-library/meltano-expand-envvars-in-array/index.md b/integration/example-library/meltano-expand-envvars-in-array/index.md
@@ -0,0 +1,19 @@
+# Expanding environment variables in elements of an array setting value
+
+This is a simple example of how to expand environment variables in elements of an array setting value.
+
+## Setup
+
+Install the project dependencies:
+
+```shell
+meltano install
+```
+
+## Run the pipeline
+
+```shell
+ANIMAL_GROUP=birds meltano invoke tap-smoke-test
+ANIMAL_GROUP=reptiles meltano invoke tap-smoke-test
+ANIMAL_GROUP=mammals meltano invoke tap-smoke-test
+```
diff --git a/integration/example-library/meltano-expand-envvars-in-array/meltano.yml b/integration/example-library/meltano-expand-envvars-in-array/meltano.yml
@@ -0,0 +1,27 @@
+version: 1
+default_environment: dev
+project_id: 95ed0825-997f-4baf-9c13-bae04ac44fe1
+environments:
+- name: dev
+- name: staging
+- name: prod
+send_anonymous_usage_stats: false
+plugins:
+  extractors:
+  - name: tap-smoke-test
+    namespace: tap_smoke_test
+    pip_url: git+https://github.com/meltano/tap-smoke-test.git
+    executable: tap-smoke-test
+    capabilities:
+    - discover
+    - catalog
+    settings:
+    - name: schema_inference_record_count
+      kind: integer
+    - name: streams
+      kind: array
+    config:
+      schema_inference_record_count: 5
+      streams:
+      - stream_name: $ANIMAL_GROUP
+        input_filename: $MELTANO_PROJECT_ROOT/data/$ANIMAL_GROUP.jsonl
diff --git a/integration/example-library/meltano-expand-envvars-in-array/output/.gitignore b/integration/example-library/meltano-expand-envvars-in-array/output/.gitignore
@@ -0,0 +1,2 @@
+*
+!.gitignore
diff --git a/src/meltano/core/utils/__init__.py b/src/meltano/core/utils/__init__.py
@@ -477,7 +477,12 @@ def __init__(self, env_var: str):
     re.VERBOSE,
 )
 
-Expandable = t.TypeVar("Expandable", str, t.Mapping[str, "Expandable"])
+Expandable = t.TypeVar(
+    "Expandable",
+    str,
+    t.Mapping[str, "Expandable"],
+    t.Sequence["Expandable"],
+)
 
 
 class EnvVarMissingBehavior(IntEnum):
@@ -523,7 +528,7 @@ def expand_env_vars(
     """  # noqa: DAR402
     if_missing = EnvVarMissingBehavior(if_missing)
 
-    if not isinstance(raw_value, (str, t.Mapping)):
+    if not isinstance(raw_value, (str, t.Mapping, list)):
         return raw_value
 
     def replacer(match: re.Match) -> str:
@@ -560,10 +565,19 @@ def _expand_env_vars(
             return {k: ENV_VAR_PATTERN.sub(replacer, v) for k, v in raw_value.items()}
         return {
             k: _expand_env_vars(v, replacer, flat)
-            if isinstance(v, (str, t.Mapping))
+            if isinstance(v, (str, t.Mapping, list))
             else v
             for k, v in raw_value.items()
         }
+    if isinstance(raw_value, list):
+        # `flat=True` doesn't seem to be used anywhere and probably doesn't make sense
+        # for lists anyway, so we don't support it here.
+        return [
+            _expand_env_vars(v, replacer, flat)
+            if isinstance(v, (str, t.Mapping, list))
+            else v
+            for v in raw_value
+        ]
     return ENV_VAR_PATTERN.sub(replacer, raw_value)
 
 

diff --git a/tests/meltano/core/test_utils.py b/tests/meltano/core/test_utils.py
@@ -161,6 +161,63 @@ def test_expand_env_vars_nested():
     assert expand_env_vars(input_dict, env) == expected_output
 
 
+@pytest.mark.parametrize(
+    ("input_array", "env", "expected_output"),
+    (
+        pytest.param(
+            [
+                {"some_key": "${ENV_VAR_1}"},
+                {"some_key": "${ENV_VAR_2}"},
+            ],
+            {"ENV_VAR_1": "substituted_1", "ENV_VAR_2": "substituted_2"},
+            [
+                {"some_key": "substituted_1"},
+                {"some_key": "substituted_2"},
+            ],
+            id="array-of-flat-dicts",
+        ),
+        pytest.param(
+            [
+                {
+                    "some_key": [
+                        {
+                            "some_key": "${ENV_VAR_1}",
+                            "another_key": "${ENV_VAR_2}",
+                        },
+                    ],
+                },
+            ],
+            {"ENV_VAR_1": "substituted_1", "ENV_VAR_2": "substituted_2"},
+            [
+                {
+                    "some_key": [
+                        {
+                            "some_key": "substituted_1",
+                            "another_key": "substituted_2",
+                        },
+                    ],
+                },
+            ],
+            id="array-of-nested-dicts",
+        ),
+        pytest.param(
+            [
+                "${ENV_VAR_1}",
+                "${ENV_VAR_2}",
+            ],
+            {"ENV_VAR_1": "substituted_1", "ENV_VAR_2": "substituted_2"},
+            [
+                "substituted_1",
+                "substituted_2",
+            ],
+            id="flat-array",
+        ),
+    ),
+)
+def test_expand_env_vars_array_nested(input_array, env, expected_output):
+    assert expand_env_vars(input_array, env) == expected_output
+
+
 def test_remove_suffix():
     assert remove_suffix("a_string", "ing") == "a_str"
     assert remove_suffix("a_string", "in") == "a_string"