📊 WDI update

dag/main.yml

@@ -530,6 +530,7 @@ steps:
     - data://garden/news/2024-05-23/gdelt_v2
 
   # World Development Indicators - WDI
+  # TO BE ARCHIVED
   data://meadow/worldbank_wdi/2024-05-20/wdi:
     - snapshot://worldbank_wdi/2024-05-20/wdi.zip
 
@@ -543,6 +544,20 @@ steps:
   data://grapher/worldbank_wdi/2024-05-20/wdi:
     - data://garden/worldbank_wdi/2024-05-20/wdi
 
+  # World Development Indicators - WDI
+  data://meadow/worldbank_wdi/2024-12-27/wdi:
+    - snapshot://worldbank_wdi/2024-12-27/wdi.zip
+
+  data://garden/worldbank_wdi/2024-12-27/wdi:
+    - snapshot://worldbank_wdi/2024-12-27/wdi.zip
+    - data://meadow/worldbank_wdi/2024-12-27/wdi
+    - data://garden/demography/2024-07-15/population
+    - data://garden/regions/2023-01-01/regions
+    - data://garden/wb/2024-07-29/income_groups
+
+  data://grapher/worldbank_wdi/2024-12-27/wdi:
+    - data://garden/worldbank_wdi/2024-12-27/wdi
+
   #
   # Aviation Safety Network - Aviation Statistics.
   #

etl/helpers.py

@@ -688,7 +688,8 @@ def get_dependency_step_name(
             short_name=short_name,
             is_private=is_private,
         )
-        matches = [dependency for dependency in self.dependencies if bool(re.match(pattern, dependency))]
+        deps = self.dependencies
+        matches = _match_dependencies(pattern, deps)
 
         # If no step was found and is_private was not specified, try again assuming step is private.
         if (len(matches) == 0) and (is_private is None):
@@ -700,7 +701,7 @@ def get_dependency_step_name(
                 short_name=short_name,
                 is_private=True,
             )
-            matches = [dependency for dependency in self.dependencies if bool(re.match(pattern, dependency))]
+            matches = _match_dependencies(pattern, self.dependencies)
 
         # If not step was found and channel is "grapher", try again assuming this is a grapher://grapher step.
         if (len(matches) == 0) and (channel == "grapher"):
@@ -712,7 +713,7 @@ def get_dependency_step_name(
                 short_name=short_name,
                 is_private=is_private,
             )
-            matches = [dependency for dependency in self.dependencies if bool(re.match(pattern, dependency))]
+            matches = _match_dependencies(pattern, self.dependencies)
 
         if len(matches) == 0:
             raise NoMatchingStepsAmongDependencies(step_name=self.step_name)
@@ -798,6 +799,11 @@ def load_mdim_config(self, filename: Optional[str] = None, path: Optional[str |
         return config
 
 
+def _match_dependencies(pattern: str, dependencies: List[str]) -> List[str]:
+    regex = re.compile(pattern)
+    return [dependency for dependency in dependencies if regex.match(dependency)]
+
+
 def print_tables_metadata_template(tables: List[Table], fields: Optional[List[str]] = None) -> None:
     # This function is meant to be used when creating code in an interactive window (or a notebook).
     # It prints a template for the metadata of the tables in the list.