fix(ingest/slack): add rate limiting to more places in slack source (d…

…atahub-project#10082)
acryldata · Mar 20, 2024 · 446fbe5 · 446fbe5
1 parent cc0552a
commit 446fbe5
Showing 1 changed file with 14 additions and 11 deletions.
diff --git a/metadata-ingestion/src/datahub/ingestion/source/slack/slack.py b/metadata-ingestion/src/datahub/ingestion/source/slack/slack.py
@@ -155,11 +155,12 @@ def _get_channel_info(
         self, cursor: Optional[str]
     ) -> Tuple[List[MetadataWorkUnit], Optional[str]]:
         result_channels: List[MetadataWorkUnit] = []
-        response = self.get_slack_client().conversations_list(
-            types="public_channel",
-            limit=self.config.channels_iteration_limit,
-            cursor=cursor,
-        )
+        with self.rate_limiter:
+            response = self.get_slack_client().conversations_list(
+                types="public_channel",
+                limit=self.config.channels_iteration_limit,
+                cursor=cursor,
+            )
         assert isinstance(response.data, dict)
         if not response.data["ok"]:
             self.report.report_failure(
@@ -240,9 +241,10 @@ def get_public_channels(self) -> Iterable[MetadataWorkUnit]:
     def populate_user_profile(self, user_obj: CorpUser) -> None:
         try:
             # https://api.slack.com/methods/users.profile.get
-            user_profile_res = self.get_slack_client().users_profile_get(
-                user=user_obj.slack_id
-            )
+            with self.rate_limiter:
+                user_profile_res = self.get_slack_client().users_profile_get(
+                    user=user_obj.slack_id
+                )
             user_profile = user_profile_res.get("profile", {})
             user_obj.title = user_profile.get("title")
             user_obj.image_url = user_profile.get("image_192")
@@ -257,9 +259,10 @@ def populate_slack_id_from_email(self, user_obj: CorpUser) -> None:
             return
         try:
             # https://api.slack.com/methods/users.lookupByEmail
-            user_info_res = self.get_slack_client().users_lookupByEmail(
-                email=user_obj.email
-            )
+            with self.rate_limiter:
+                user_info_res = self.get_slack_client().users_lookupByEmail(
+                    email=user_obj.email
+                )
             user_info = user_info_res.get("user", {})
             user_obj.slack_id = user_info.get("id")
         except Exception as e: