From 580390ed82e49076527ab574bf1f0c3d7dd2f4b7 Mon Sep 17 00:00:00 2001 From: Aseem Bansal Date: Mon, 2 Dec 2024 19:31:59 +0530 Subject: [PATCH] fix(ingest/gc): delete invalid dpis (#11998) --- .../src/datahub/ingestion/source/gc/dataprocess_cleanup.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/metadata-ingestion/src/datahub/ingestion/source/gc/dataprocess_cleanup.py b/metadata-ingestion/src/datahub/ingestion/source/gc/dataprocess_cleanup.py index 0f35e1a67fede7..ca67cd6daa045b 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/gc/dataprocess_cleanup.py +++ b/metadata-ingestion/src/datahub/ingestion/source/gc/dataprocess_cleanup.py @@ -277,7 +277,12 @@ def delete_dpi_from_datajobs(self, job: DataJobEntity) -> None: assert self.ctx.graph dpis = self.fetch_dpis(job.urn, self.config.batch_size) - dpis.sort(key=lambda x: x["created"]["time"], reverse=True) + dpis.sort( + key=lambda x: x["created"]["time"] + if x["created"] and x["created"]["time"] + else 0, + reverse=True, + ) with ThreadPoolExecutor(max_workers=self.config.max_workers) as executor: if self.config.keep_last_n: