Merge pull request #225 from ASFHyP3/develop

Release: remove description and revise DEM coverage checks

CHANGELOG.md

@@ -4,6 +4,12 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
+## [0.4.0]
+### Changed
+- Implemented 0.15° buffer and 20% threshold in DEM coverage checks when submitting new jobs.  As a result slightly more granules will be rejected as having insufficient coverage.
+
+### Removed
+- Removed optional `description` field for jobs
 
 ## [0.3.9]
 ### Added

api/src/hyp3_api/openapi-spec.yml

@@ -141,8 +141,6 @@ components:
           $ref: "#/components/schemas/job_type"
         job_parameters:
           $ref: "#/components/schemas/job_parameters"
-        description:
-          $ref: "#/components/schemas/description"
         name:
           $ref: "#/components/schemas/name"
 
@@ -175,8 +173,6 @@ components:
           $ref: "#/components/schemas/datetime"
         status_code:
           $ref: "#/components/schemas/status_code"
-        description:
-          $ref: "#/components/schemas/description"
         name:
           $ref: "#/components/schemas/name"
         files:
@@ -307,12 +303,6 @@ components:
         - FAILED
       example: SUCCEEDED
 
-    description:
-      description: User provided text to describe the job
-      type: string
-      minLength: 1
-      example: a description of the job
-
     name:
       description: User provided text to name the job
       type: string

api/src/hyp3_api/validation.py

@@ -2,7 +2,7 @@
 import os
 
 import requests
-from shapely.geometry import Polygon, shape
+from shapely.geometry import MultiPolygon, Polygon, shape
 
 from hyp3_api import CMR_URL
 
@@ -13,14 +13,15 @@ class GranuleValidationError(Exception):
     pass
 
 
-def check_intersects_with_coverage(granule: Polygon):
+def has_sufficient_coverage(granule: Polygon, buffer: float = 0.15, threshold: float = 0.2):
     global DEM_COVERAGE
     if DEM_COVERAGE is None:
-        DEM_COVERAGE = get_coverage_shapes_from_geojson()
-    for polygon in DEM_COVERAGE:
-        if granule.intersects(polygon):
-            return True
-    return False
+        DEM_COVERAGE = MultiPolygon(get_coverage_shapes_from_geojson())
+
+    buffered_granule = granule.buffer(buffer)
+    covered_area = buffered_granule.intersection(DEM_COVERAGE).area
+
+    return covered_area / buffered_granule.area >= threshold
 
 
 def validate_granules(granules):
@@ -63,8 +64,7 @@ def check_granules_exist(granules, granule_metadata):
 
 
 def check_dem_coverage(granule_metadata):
-    bad_granules = {granule['name'] for granule in granule_metadata if
-                    not check_intersects_with_coverage(granule['polygon'])}
+    bad_granules = {granule['name'] for granule in granule_metadata if not has_sufficient_coverage(granule['polygon'])}
     if bad_granules:
         raise GranuleValidationError(f'Some requested scenes do not have DEM coverage: {", ".join(bad_granules)}')
 

api/tests/conftest.py

@@ -46,7 +46,6 @@ def get_table_properties_from_template():
 
 
 def make_job(granule='S1B_IW_SLC__1SDV_20200604T082207_20200604T082234_021881_029874_5E38',
-             description='someDescription',
              name='someName',
              job_type='RTC_GAMMA'):
     job = {
@@ -55,8 +54,6 @@ def make_job(granule='S1B_IW_SLC__1SDV_20200604T082207_20200604T082234_021881_02
             'granule': granule
         }
     }
-    if description is not None:
-        job['description'] = description
     if name is not None:
         job['name'] = name
 

api/tests/test_check_dem.py

@@ -2,7 +2,7 @@
 from shapely.geometry import Polygon
 
 
-from hyp3_api.validation import GranuleValidationError, check_dem_coverage
+from hyp3_api.validation import GranuleValidationError, check_dem_coverage, has_sufficient_coverage
 
 
 def nsew(north, south, east, west):
@@ -15,40 +15,67 @@ def test_check_dem_coverage():
             'polygon': nsew(45, 41, -104, -111),
             'name': 'good1',
         },
-        {  # Alluetean Islands over antimeridian
+        {  # completely covered Aleutian Islands over antimeridian; should pass with fixed antimeridian
             'polygon': nsew(51.7, 51.3, 179.7, -179.3),
             'name': 'good2',
         },
-        {  # ocean over antimeridian; no dem coverage but we expect to pass validation
-            'polygon': nsew(-40, -41, 179.7, -179.3),
+        {  # not enough coverage of Aleutian Islands over antimeridian
+            # NOTE: Passes today but should FAIL with antimeridian feature fix
+            'polygon': nsew(51.7, 41.3, 179.7, -179.3),
             'name': 'good3',
         },
         {  # completely encloses tile over Ascension Island in the Atlantic
-            'polygon': nsew(-6, -9, -16, -13),
+            'polygon': nsew(-6, -9, -15, -14),
             'name': 'good4',
         },
+        {  # minimum intersects off the coast of Eureka, CA
+            'polygon': nsew(40.1, 40, -126, -124.845),
+            'name': 'good5',
+        },
+        {  # almost minimum intersects off the coast of Eureka, CA
+            'polygon': nsew(40.1, 40, -126, -124.849),
+            'name': 'bad1',
+        },
         {  # barely intersects off the coast of Eureka, CA
             'polygon': nsew(40.1, 40, -126, -125.00166),
-            'name': 'good5',
+            'name': 'bad2',
         },
         {  # barely misses off the coast of Eureka, CA
             'polygon': nsew(40.1, 40, -126, -125.00167),
-            'name': 'bad1',
+            'name': 'bad3',
         },
         {  # polygon in missing tile over Gulf of Californa
             'polygon': nsew(26.9, 26.1, -110.1, -110.9),
-            'name': 'bad2',
+            'name': 'bad4',
         },
         {  # southern Greenland
             'polygon': nsew(62, 61, -44, -45),
-            'name': 'bad3',
+            'name': 'bad5',
         },
         {  # Antarctica
             'polygon': nsew(-62, -90, 180, -180),
-            'name': 'bad4',
+            'name': 'bad6',
+        },
+        {  # ocean over antimeridian; no dem coverage and also not enough wraparound land intersection
+            'polygon': nsew(-40, -41, 179.7, -179.3),
+            'name': 'bad7',
         },
     ]
     with raises(GranuleValidationError) as exception_info:
         check_dem_coverage(polygons)
     for polygon in polygons:
         assert polygon['name'].startswith('bad') == (polygon['name'] in str(exception_info))
+
+
+def test_has_sufficient_coverage_buffer():
+    needs_buffer = nsew(40.1, 40, -126, -124.845)
+    assert has_sufficient_coverage(needs_buffer)
+    assert has_sufficient_coverage(needs_buffer, buffer=0.16)
+    assert not has_sufficient_coverage(needs_buffer, buffer=0.14)
+
+
+def test_has_sufficient_coverage_threshold():
+    poly = nsew(40.1, 40, -126, -124.845)
+    assert has_sufficient_coverage(poly)
+    assert has_sufficient_coverage(poly, threshold=0.19)
+    assert not has_sufficient_coverage(poly, threshold=0.21)

api/tests/test_submit_job.py

@@ -63,27 +63,6 @@ def test_submit_without_jobs(client):
     assert response.status_code == status.HTTP_400_BAD_REQUEST
 
 
-def test_submit_job_without_description(client, table):
-    login(client)
-    batch = [
-        make_job(description=None)
-    ]
-    setup_requests_mock(batch)
-
-    response = submit_batch(client, batch)
-    assert response.status_code == status.HTTP_200_OK
-
-
-def test_submit_job_with_empty_description(client):
-    login(client)
-    batch = [
-        make_job(description='')
-    ]
-    setup_requests_mock(batch)
-    response = submit_batch(client, batch)
-    assert response.status_code == status.HTTP_400_BAD_REQUEST
-
-
 def test_submit_job_without_name(client, table):
     login(client)
     batch = [