Merge pull request #374 from ASFHyP3/develop

Release get-files changes to ignore unzipped product files

CHANGELOG.md

@@ -5,6 +5,11 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
 
+## [0.8.16]
+### Changed
+- `get_files.py` now only includes product files ending in `.zip` or `.nc` in the `files` list returned
+  in `GET /jobs` API responses
+
 ## [0.8.15]
 ### Changed
 - S3 content bucket now allows public `s3:ListBucket` and `s3:GetObjectTagging`

apps/get-files/src/get_files.py

@@ -1,6 +1,8 @@
 from datetime import datetime
 from os import environ
 from os.path import basename
+from pathlib import Path
+from typing import Union
 
 import boto3
 
@@ -27,13 +29,17 @@ def get_object_file_type(bucket, key):
     return None
 
 
+def visible_product(product_path: Union[str, Path]) -> bool:
+    return Path(product_path).suffix in ('.zip', '.nc')
+
+
 def get_products(files):
     return [{
         'url': item['download_url'],
         'size': item['size'],
         'filename': item['filename'],
         's3': item['s3'],
-    } for item in files if item['file_type'] == 'product']
+    } for item in files if item['file_type'] == 'product' and visible_product(item['filename'])]
 
 
 def get_file_urls_by_type(file_list, file_type):

tests/test_get_files.py

@@ -1,7 +1,9 @@
+from pathlib import Path
+
 import pytest
 from botocore.stub import Stubber
 
-from get_files import S3_CLIENT, get_download_url, get_expiration_time, get_object_file_type, lambda_handler
+import get_files
 
 
 @pytest.fixture(autouse=True)
@@ -12,13 +14,13 @@ def setup_env(monkeypatch):
 
 @pytest.fixture
 def s3_stubber():
-    with Stubber(S3_CLIENT) as stubber:
+    with Stubber(get_files.S3_CLIENT) as stubber:
         yield stubber
         stubber.assert_no_pending_responses()
 
 
 def test_get_download_url():
-    assert get_download_url('myBucket', 'myKey') == 'https://myBucket.s3.myRegion.amazonaws.com/myKey'
+    assert get_files.get_download_url('myBucket', 'myKey') == 'https://myBucket.s3.myRegion.amazonaws.com/myKey'
 
 
 def stub_expiration(s3_stubber: Stubber, bucket, key):
@@ -35,7 +37,7 @@ def stub_expiration(s3_stubber: Stubber, bucket, key):
 
 def test_get_expiration(s3_stubber: Stubber):
     stub_expiration(s3_stubber, 'myBucket', 'myKey')
-    response = get_expiration_time('myBucket', 'myKey')
+    response = get_files.get_expiration_time('myBucket', 'myKey')
     assert response == '2020-01-01T00:00:00+00:00'
 
 
@@ -57,10 +59,24 @@ def stub_get_object_tagging(s3_stubber: Stubber, bucket, key, file_type):
 
 def test_get_file_type(s3_stubber: Stubber):
     stub_get_object_tagging(s3_stubber, 'myBucket', 'myKey', file_type='product')
-    response = get_object_file_type('myBucket', 'myKey')
+    response = get_files.get_object_file_type('myBucket', 'myKey')
     assert response == 'product'
 
 
+def test_visible_product():
+    assert not get_files.visible_product('myFile.tif')
+    assert not get_files.visible_product('myFile.xml')
+    assert not get_files.visible_product('myFile.md.txt')
+    assert get_files.visible_product('myFile.zip')
+    assert get_files.visible_product('myFile.nc')
+
+    assert not get_files.visible_product(Path('myFile.tif'))
+    assert not get_files.visible_product(Path('myFile.xml'))
+    assert not get_files.visible_product(Path('myFile.md.txt'))
+    assert get_files.visible_product(Path('myFile.zip'))
+    assert get_files.visible_product(Path('myFile.nc'))
+
+
 def stub_list_files(s3_stubber: Stubber, job_id, bucket, contents):
     params = {
         'Bucket': bucket,
@@ -72,12 +88,16 @@ def stub_list_files(s3_stubber: Stubber, job_id, bucket, contents):
     s3_stubber.add_response('list_objects_v2', expected_params=params, service_response=s3_response)
 
 
-def test_get_files(s3_stubber: Stubber):
+def test_get_files_zipped_product(s3_stubber: Stubber):
     files = [
         {
             'Key': 'myJobId/myProduct.zip',
             'Size': 50,
         },
+        {
+            'Key': 'myJobId/myProduct.tif',
+            'Size': 30,
+        },
         {
             'Key': 'myJobId/myThumbnail.png',
             'Size': 5,
@@ -97,6 +117,7 @@ def test_get_files(s3_stubber: Stubber):
     ]
     stub_list_files(s3_stubber, 'myJobId', 'myBucket', files)
     stub_get_object_tagging(s3_stubber, 'myBucket', 'myJobId/myProduct.zip', 'product')
+    stub_get_object_tagging(s3_stubber, 'myBucket', 'myJobId/myProduct.tif', 'product')
     stub_get_object_tagging(s3_stubber, 'myBucket', 'myJobId/myThumbnail.png', 'amp_thumbnail')
     stub_get_object_tagging(s3_stubber, 'myBucket', 'myJobId/myBrowse.png', 'amp_browse')
     stub_get_object_tagging(s3_stubber, 'myBucket', 'myJobId/myBrowse_rgb.png', 'rgb_browse')
@@ -106,7 +127,7 @@ def test_get_files(s3_stubber: Stubber):
     event = {
         'job_id': 'myJobId'
     }
-    response = lambda_handler(event, None)
+    response = get_files.lambda_handler(event, None)
     assert response == {
         'expiration_time': '2020-01-01T00:00:00+00:00',
         'files': [
@@ -127,3 +148,54 @@ def test_get_files(s3_stubber: Stubber):
         'thumbnail_images': ['https://myBucket.s3.myRegion.amazonaws.com/myJobId/myThumbnail.png'],
         'logs': ['https://myBucket.s3.myRegion.amazonaws.com/myJobId/myJobId.log'],
     }
+
+
+def test_get_files_netcdf_product(s3_stubber: Stubber):
+    files = [
+        {
+            'Key': 'myJobId/myProduct.nc',
+            'Size': 50,
+        },
+        {
+            'Key': 'myJobId/myThumbnail.png',
+            'Size': 5,
+        },
+        {
+            'Key': 'myJobId/myBrowse.png',
+            'Size': 10,
+        },
+        {
+            'Key': 'myJobId/myJobId.log',
+            'Size': 10,
+        },
+    ]
+    stub_list_files(s3_stubber, 'myJobId', 'myBucket', files)
+    stub_get_object_tagging(s3_stubber, 'myBucket', 'myJobId/myProduct.nc', 'product')
+    stub_get_object_tagging(s3_stubber, 'myBucket', 'myJobId/myThumbnail.png', 'amp_thumbnail')
+    stub_get_object_tagging(s3_stubber, 'myBucket', 'myJobId/myBrowse.png', 'amp_browse')
+    stub_get_object_tagging(s3_stubber, 'myBucket', 'myJobId/myJobId.log', 'log')
+    stub_expiration(s3_stubber, 'myBucket', 'myJobId/myJobId.log')
+
+    event = {
+        'job_id': 'myJobId'
+    }
+    response = get_files.lambda_handler(event, None)
+    assert response == {
+        'expiration_time': '2020-01-01T00:00:00+00:00',
+        'files': [
+            {
+                'url': 'https://myBucket.s3.myRegion.amazonaws.com/myJobId/myProduct.nc',
+                's3': {
+                    'bucket': 'myBucket',
+                    'key': 'myJobId/myProduct.nc',
+                },
+                'size': 50,
+                'filename': 'myProduct.nc',
+            }
+        ],
+        'browse_images': [
+            'https://myBucket.s3.myRegion.amazonaws.com/myJobId/myBrowse.png',
+        ],
+        'thumbnail_images': ['https://myBucket.s3.myRegion.amazonaws.com/myJobId/myThumbnail.png'],
+        'logs': ['https://myBucket.s3.myRegion.amazonaws.com/myJobId/myJobId.log'],
+    }