feat: support multiple fields in joinkey

chenxuanrong · Oct 17, 2023 · 6af7c83 · 6af7c83
1 parent eabb10b
commit 6af7c83
Show file tree

Hide file tree

Showing 7 changed files with 131 additions and 140 deletions.
diff --git a/piperider_cli/cli.py b/piperider_cli/cli.py
@@ -395,6 +395,6 @@ def reconcile(**kwargs):
 
 
 if __name__ == '__main__':
-    # reconciler = Reconciler(engine=None)
-    # reconciler.reconcile(project='pipeline_v2')
-    compare_reports()
+    reconciler = Reconciler(engine=None)
+    reconciler.reconcile(project='test_mock')
+    # compare_reports()
diff --git a/piperider_cli/configuration.py b/piperider_cli/configuration.py
@@ -218,7 +218,8 @@ def load(cls, piperider_config_path=PIPERIDER_CONFIG_PATH, piperider_reconcile_p
         if os.path.exists(piperider_reconcile_path):
             # logger.info(f'Loading reconcile rules from {piperider_reconcile_path}')
             with open(piperider_reconcile_path, 'r') as frecon:
-                r_config = yaml.safe_load(frecon)
+                yml=yaml.YAML(typ='safe', pure=True)                
+                r_config = yml.load(frecon)
 
             r_projects = r_config.get('Reconciles')
             if r_projects:

diff --git a/piperider_cli/reconciler/reconcile_rule.py b/piperider_cli/reconciler/reconcile_rule.py
@@ -23,12 +23,6 @@ class ReconcileSuite:
     target_join_key: str    
     column_reconcile_rules: List[ColumnReconcileRule]
     description: str = None
-    # base_source: str
-    # target_source: str
-    # base_table: str
-    # target_table: str
-    # base_join_key: str
-    # target_join_key: str
 
     def validate(self):
         reasons = []