From 05d362a94e140d508019d686ccf29b010db5e40d Mon Sep 17 00:00:00 2001 From: Aseem Bansal Date: Wed, 20 Nov 2024 20:47:39 +0530 Subject: [PATCH] fix(ingest): add error handling (#11905) --- .../src/datahub/ingestion/source/gc/dataprocess_cleanup.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/metadata-ingestion/src/datahub/ingestion/source/gc/dataprocess_cleanup.py b/metadata-ingestion/src/datahub/ingestion/source/gc/dataprocess_cleanup.py index 80f7b7a9f4480c..130f2c9c2e12fc 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/gc/dataprocess_cleanup.py +++ b/metadata-ingestion/src/datahub/ingestion/source/gc/dataprocess_cleanup.py @@ -401,7 +401,10 @@ def get_workunits_internal(self) -> Iterable[MetadataWorkUnit]: total_runs=job.get("entity").get("runs").get("total"), ) if datajob_entity.total_runs > 0: - self.delete_dpi_from_datajobs(datajob_entity) + try: + self.delete_dpi_from_datajobs(datajob_entity) + except Exception as e: + logger.error(f"While trying to delete {datajob_entity} got {e}") if ( datajob_entity.total_runs == 0 and self.config.delete_empty_data_jobs