Merge pull request #493 from wri/develop

GTC-2449 Catch access denied error in get_aws_files() [merge to master]

app/routes/datasets/versions.py

@@ -560,7 +560,7 @@ def _verify_source_file_access(sources: List[str]) -> None:
         raise HTTPException(
             status_code=400,
             detail=(
-                "Cannot access all of the source files. "
+                "Cannot access all of the source files (non-existent or access denied). "
                 f"Invalid sources: {invalid_sources}"
             ),
         )

app/utils/aws.py

@@ -1,8 +1,10 @@
 from typing import Any, Dict, List, Optional, Sequence
 
 import boto3
+import botocore
 import httpx
 from httpx_auth import AWS4Auth
+from fastapi.logger import logger
 
 from ..settings.globals import (
     AWS_REGION,
@@ -108,7 +110,11 @@ def get_aws_files(
             if exit_after_max and num_matches >= exit_after_max:
                 break
 
-    except s3_client.exceptions.NoSuchBucket:
+    # Strangely, s3_client.exceptions has NoSuchBucket, but doesn't have
+    # AccessDenied, even though you can get that error, so we just catch all botocore
+    # exceptions.
+    except botocore.exceptions.ClientError as error:
+        logger.warning(f"get_aws_file: {error}")
         matches = list()
 
     return matches

tests/routes/datasets/test_versions.py

@@ -310,7 +310,7 @@ async def test_invalid_source_uri(async_client: AsyncClient):
     assert response.json()["status"] == "failed"
     assert (
         response.json()["message"]
-        == f"Cannot access all of the source files. Invalid sources: ['{bad_uri}']"
+        == f"Cannot access all of the source files (non-existent or access denied). Invalid sources: ['{bad_uri}']"
     )
 
     # Create a version with a valid source_uri so we have something to append to
@@ -332,7 +332,7 @@ async def test_invalid_source_uri(async_client: AsyncClient):
     assert response.json()["status"] == "failed"
     assert (
         response.json()["message"]
-        == f"Cannot access all of the source files. Invalid sources: ['{bad_uri}']"
+        == f"Cannot access all of the source files (non-existent or access denied). Invalid sources: ['{bad_uri}']"
     )
 
     # Test appending to a version that DOESN'T exist