fix(ingest/databricks): Pinning databricks sdk to not fail on mypy is…

…sues (datahub-project#9500)
ethan-cartwright · Dec 21, 2023 · cfc641f · cfc641f
1 parent a18c720
commit cfc641f
Show file tree

Hide file tree

Showing 2 changed files with 4 additions and 2 deletions.
diff --git a/metadata-ingestion/setup.py b/metadata-ingestion/setup.py
@@ -274,7 +274,9 @@
 
 databricks = {
     # 0.1.11 appears to have authentication issues with azure databricks
-    "databricks-sdk>=0.9.0",
+    # 0.16.0 added py.typed support which caused mypy to fail. The databricks sdk is pinned until we resolve mypy issues.
+    # https://github.com/databricks/databricks-sdk-py/pull/483
+    "databricks-sdk>=0.9.0,<0.16.0",
     "pyspark~=3.3.0",
     "requests",
     # Version 2.4.0 includes sqlalchemy dialect, 2.8.0 includes some bug fixes

diff --git a/metadata-ingestion/src/datahub/ingestion/source/aws/aws_common.py b/metadata-ingestion/src/datahub/ingestion/source/aws/aws_common.py
@@ -167,7 +167,7 @@ def get_session(self) -> Session:
 
         return session
 
-    def get_credentials(self) -> Dict[str, str]:
+    def get_credentials(self) -> Dict[str, Optional[str]]:
         credentials = self.get_session().get_credentials()
         if credentials is not None:
             return {