From 276c26c2927d47d5f41edc1b7c242e53f0f2f93f Mon Sep 17 00:00:00 2001 From: Andrew Johnston Date: Wed, 12 Aug 2020 10:45:47 -0800 Subject: [PATCH 1/9] remove optional description field for jobs --- CHANGELOG.md | 3 +++ api/src/hyp3_api/openapi-spec.yml | 10 ---------- api/tests/conftest.py | 3 --- api/tests/test_submit_job.py | 21 --------------------- 4 files changed, 3 insertions(+), 34 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 2c491c556..363f9fed4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,9 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## [Unreleased] +### Removed +- Removed optional `description` field for jobs ## [0.3.9] ### Added diff --git a/api/src/hyp3_api/openapi-spec.yml b/api/src/hyp3_api/openapi-spec.yml index cda26cc1f..606071ccf 100644 --- a/api/src/hyp3_api/openapi-spec.yml +++ b/api/src/hyp3_api/openapi-spec.yml @@ -141,8 +141,6 @@ components: $ref: "#/components/schemas/job_type" job_parameters: $ref: "#/components/schemas/job_parameters" - description: - $ref: "#/components/schemas/description" name: $ref: "#/components/schemas/name" @@ -175,8 +173,6 @@ components: $ref: "#/components/schemas/datetime" status_code: $ref: "#/components/schemas/status_code" - description: - $ref: "#/components/schemas/description" name: $ref: "#/components/schemas/name" files: @@ -307,12 +303,6 @@ components: - FAILED example: SUCCEEDED - description: - description: User provided text to describe the job - type: string - minLength: 1 - example: a description of the job - name: description: User provided text to name the job type: string diff --git a/api/tests/conftest.py b/api/tests/conftest.py index 491d6d4f5..305c2a49c 100644 --- a/api/tests/conftest.py +++ b/api/tests/conftest.py @@ -46,7 +46,6 @@ def get_table_properties_from_template(): def make_job(granule='S1B_IW_SLC__1SDV_20200604T082207_20200604T082234_021881_029874_5E38', - description='someDescription', name='someName', job_type='RTC_GAMMA'): job = { @@ -55,8 +54,6 @@ def make_job(granule='S1B_IW_SLC__1SDV_20200604T082207_20200604T082234_021881_02 'granule': granule } } - if description is not None: - job['description'] = description if name is not None: job['name'] = name diff --git a/api/tests/test_submit_job.py b/api/tests/test_submit_job.py index 62b6b0d73..dc13db178 100644 --- a/api/tests/test_submit_job.py +++ b/api/tests/test_submit_job.py @@ -63,27 +63,6 @@ def test_submit_without_jobs(client): assert response.status_code == status.HTTP_400_BAD_REQUEST -def test_submit_job_without_description(client, table): - login(client) - batch = [ - make_job(description=None) - ] - setup_requests_mock(batch) - - response = submit_batch(client, batch) - assert response.status_code == status.HTTP_200_OK - - -def test_submit_job_with_empty_description(client): - login(client) - batch = [ - make_job(description='') - ] - setup_requests_mock(batch) - response = submit_batch(client, batch) - assert response.status_code == status.HTTP_400_BAD_REQUEST - - def test_submit_job_without_name(client, table): login(client) batch = [ From f3045824f62414c295254cf3a7cbce154696aa58 Mon Sep 17 00:00:00 2001 From: Andrew Johnston Date: Thu, 13 Aug 2020 16:01:13 -0800 Subject: [PATCH 2/9] add buffer and threshold to dem coverage check --- api/src/hyp3_api/validation.py | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/api/src/hyp3_api/validation.py b/api/src/hyp3_api/validation.py index 89f6a364e..7c614d0ed 100644 --- a/api/src/hyp3_api/validation.py +++ b/api/src/hyp3_api/validation.py @@ -13,14 +13,18 @@ class GranuleValidationError(Exception): pass -def check_intersects_with_coverage(granule: Polygon): +def has_sufficient_coverage(granule: Polygon, buffer: float = 0.15, threshold: float = 0.2): global DEM_COVERAGE if DEM_COVERAGE is None: DEM_COVERAGE = get_coverage_shapes_from_geojson() + + buffered_granule = granule.buffer(buffer) + covered_area = 0 + for polygon in DEM_COVERAGE: - if granule.intersects(polygon): - return True - return False + covered_area += buffered_granule.intersection(polygon).area + + return covered_area / buffered_granule.area >= threshold def validate_granules(granules): @@ -63,8 +67,7 @@ def check_granules_exist(granules, granule_metadata): def check_dem_coverage(granule_metadata): - bad_granules = {granule['name'] for granule in granule_metadata if - not check_intersects_with_coverage(granule['polygon'])} + bad_granules = {granule['name'] for granule in granule_metadata if not has_sufficient_coverage(granule['polygon'])} if bad_granules: raise GranuleValidationError(f'Some requested scenes do not have DEM coverage: {", ".join(bad_granules)}') From 6f7ef9d109ce021701d70679a440ec2a828d141b Mon Sep 17 00:00:00 2001 From: Andrew Johnston Date: Thu, 13 Aug 2020 16:09:12 -0800 Subject: [PATCH 3/9] add todo to revise dem tests --- api/tests/test_check_dem.py | 25 +++++++++++++------------ 1 file changed, 13 insertions(+), 12 deletions(-) diff --git a/api/tests/test_check_dem.py b/api/tests/test_check_dem.py index 33d264dfa..4fa23c3d5 100644 --- a/api/tests/test_check_dem.py +++ b/api/tests/test_check_dem.py @@ -19,18 +19,19 @@ def test_check_dem_coverage(): 'polygon': nsew(51.7, 51.3, 179.7, -179.3), 'name': 'good2', }, - { # ocean over antimeridian; no dem coverage but we expect to pass validation - 'polygon': nsew(-40, -41, 179.7, -179.3), - 'name': 'good3', - }, - { # completely encloses tile over Ascension Island in the Atlantic - 'polygon': nsew(-6, -9, -16, -13), - 'name': 'good4', - }, - { # barely intersects off the coast of Eureka, CA - 'polygon': nsew(40.1, 40, -126, -125.00166), - 'name': 'good5', - }, + # TODO update for new buffer, threshold rules + # { # ocean over antimeridian; no dem coverage but we expect to pass validation + # 'polygon': nsew(-40, -41, 179.7, -179.3), + # 'name': 'good3', + # }, + # { # completely encloses tile over Ascension Island in the Atlantic + # 'polygon': nsew(-6, -9, -16, -13), + # 'name': 'good4', + # }, + # { # barely intersects off the coast of Eureka, CA + # 'polygon': nsew(40.1, 40, -126, -125.00166), + # 'name': 'good5', + # }, { # barely misses off the coast of Eureka, CA 'polygon': nsew(40.1, 40, -126, -125.00167), 'name': 'bad1', From 08b5dfd6e7489f7e9ca101937779e310f5a3a63e Mon Sep 17 00:00:00 2001 From: Andrew Johnston Date: Thu, 13 Aug 2020 16:10:45 -0800 Subject: [PATCH 4/9] initialize covered_area as float --- api/src/hyp3_api/validation.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api/src/hyp3_api/validation.py b/api/src/hyp3_api/validation.py index 7c614d0ed..4fba42de0 100644 --- a/api/src/hyp3_api/validation.py +++ b/api/src/hyp3_api/validation.py @@ -19,7 +19,7 @@ def has_sufficient_coverage(granule: Polygon, buffer: float = 0.15, threshold: f DEM_COVERAGE = get_coverage_shapes_from_geojson() buffered_granule = granule.buffer(buffer) - covered_area = 0 + covered_area = 0.0 for polygon in DEM_COVERAGE: covered_area += buffered_granule.intersection(polygon).area From 240f7ed6c53e66e17a83f73410b08ce57e16bdd3 Mon Sep 17 00:00:00 2001 From: Joseph H Kennedy Date: Fri, 14 Aug 2020 16:32:35 -0800 Subject: [PATCH 5/9] Update tests --- api/tests/test_check_dem.py | 63 ++++++++++++++++++++++++++----------- 1 file changed, 44 insertions(+), 19 deletions(-) diff --git a/api/tests/test_check_dem.py b/api/tests/test_check_dem.py index 4fa23c3d5..1028135a7 100644 --- a/api/tests/test_check_dem.py +++ b/api/tests/test_check_dem.py @@ -2,7 +2,7 @@ from shapely.geometry import Polygon -from hyp3_api.validation import GranuleValidationError, check_dem_coverage +from hyp3_api.validation import GranuleValidationError, check_dem_coverage, has_sufficient_coverage def nsew(north, south, east, west): @@ -15,41 +15,66 @@ def test_check_dem_coverage(): 'polygon': nsew(45, 41, -104, -111), 'name': 'good1', }, - { # Alluetean Islands over antimeridian + { # completely covered Aleutian Islands over antimeridian; should pass with fixed antimeridian 'polygon': nsew(51.7, 51.3, 179.7, -179.3), 'name': 'good2', }, - # TODO update for new buffer, threshold rules - # { # ocean over antimeridian; no dem coverage but we expect to pass validation - # 'polygon': nsew(-40, -41, 179.7, -179.3), - # 'name': 'good3', - # }, - # { # completely encloses tile over Ascension Island in the Atlantic - # 'polygon': nsew(-6, -9, -16, -13), - # 'name': 'good4', - # }, - # { # barely intersects off the coast of Eureka, CA - # 'polygon': nsew(40.1, 40, -126, -125.00166), - # 'name': 'good5', - # }, + { # not enough coverage of Aleutian Islands over antimeridian; should FAIL with fixed antimeridian + 'polygon': nsew(51.7, 51.3, 179.7, -179.3), + 'name': 'shouldbebad1', + }, + { # completely encloses tile over Ascension Island in the Atlantic + 'polygon': nsew(-6, -9, -15, -14), + 'name': 'good4', + }, + { # minimum intersects off the coast of Eureka, CA + 'polygon': nsew(40.1, 40, -126, -124.845), + 'name': 'good5', + }, + { # almost minimum intersects off the coast of Eureka, CA + 'polygon': nsew(40.1, 40, -126, -124.849), + 'name': 'bad1', + }, + { # barely intersects off the coast of Eureka, CA + 'polygon': nsew(40.1, 40, -126, -125.00166), + 'name': 'bad2', + }, { # barely misses off the coast of Eureka, CA 'polygon': nsew(40.1, 40, -126, -125.00167), - 'name': 'bad1', + 'name': 'bad3', }, { # polygon in missing tile over Gulf of Californa 'polygon': nsew(26.9, 26.1, -110.1, -110.9), - 'name': 'bad2', + 'name': 'bad4', }, { # southern Greenland 'polygon': nsew(62, 61, -44, -45), - 'name': 'bad3', + 'name': 'bad5', }, { # Antarctica 'polygon': nsew(-62, -90, 180, -180), - 'name': 'bad4', + 'name': 'bad6', + }, + { # ocean over antimeridian; no dem coverage and also not enough wraparound land intersection + 'polygon': nsew(-40, -41, 179.7, -179.3), + 'name': 'bad7', }, ] with raises(GranuleValidationError) as exception_info: check_dem_coverage(polygons) for polygon in polygons: assert polygon['name'].startswith('bad') == (polygon['name'] in str(exception_info)) + + +def test_has_sufficient_coverage_buffer(): + needs_buffer = nsew(40.1, 40, -126, -124.845) + assert has_sufficient_coverage(needs_buffer) + assert has_sufficient_coverage(needs_buffer, buffer=0.16) + assert not has_sufficient_coverage(needs_buffer, buffer=0.14) + + +def test_has_sufficient_coverage_threshold(): + poly = nsew(40.1, 40, -126, -124.845) + assert has_sufficient_coverage(poly) + assert has_sufficient_coverage(poly, threshold=0.19) + assert not has_sufficient_coverage(poly, threshold=0.21) From 22aa6b7098e6ce0b0d3e09975f3afde9330761bc Mon Sep 17 00:00:00 2001 From: Joseph H Kennedy Date: Fri, 14 Aug 2020 16:41:10 -0800 Subject: [PATCH 6/9] Use MultiPolygon to check coverage --- api/src/hyp3_api/validation.py | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/api/src/hyp3_api/validation.py b/api/src/hyp3_api/validation.py index 4fba42de0..c26f713b1 100644 --- a/api/src/hyp3_api/validation.py +++ b/api/src/hyp3_api/validation.py @@ -2,7 +2,7 @@ import os import requests -from shapely.geometry import Polygon, shape +from shapely.geometry import MultiPolygon, Polygon, shape from hyp3_api import CMR_URL @@ -16,13 +16,10 @@ class GranuleValidationError(Exception): def has_sufficient_coverage(granule: Polygon, buffer: float = 0.15, threshold: float = 0.2): global DEM_COVERAGE if DEM_COVERAGE is None: - DEM_COVERAGE = get_coverage_shapes_from_geojson() + DEM_COVERAGE = MultiPolygon(get_coverage_shapes_from_geojson()) buffered_granule = granule.buffer(buffer) - covered_area = 0.0 - - for polygon in DEM_COVERAGE: - covered_area += buffered_granule.intersection(polygon).area + covered_area = buffered_granule.intersection(DEM_COVERAGE).area return covered_area / buffered_granule.area >= threshold From 90f6042bb7fd1e32916a242053bbcbffa1056f58 Mon Sep 17 00:00:00 2001 From: Joseph H Kennedy Date: Fri, 14 Aug 2020 16:49:33 -0800 Subject: [PATCH 7/9] Fix sbouldbebad1 --- api/tests/test_check_dem.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api/tests/test_check_dem.py b/api/tests/test_check_dem.py index 1028135a7..fedbad734 100644 --- a/api/tests/test_check_dem.py +++ b/api/tests/test_check_dem.py @@ -20,7 +20,7 @@ def test_check_dem_coverage(): 'name': 'good2', }, { # not enough coverage of Aleutian Islands over antimeridian; should FAIL with fixed antimeridian - 'polygon': nsew(51.7, 51.3, 179.7, -179.3), + 'polygon': nsew(51.7, 41.3, 179.7, -179.3), 'name': 'shouldbebad1', }, { # completely encloses tile over Ascension Island in the Atlantic From ab279b48e337aacf272129447b4e02043b05fe04 Mon Sep 17 00:00:00 2001 From: Joseph H Kennedy Date: Fri, 14 Aug 2020 16:56:28 -0800 Subject: [PATCH 8/9] Update test comments for future fix --- api/tests/test_check_dem.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/api/tests/test_check_dem.py b/api/tests/test_check_dem.py index fedbad734..f3448b6cf 100644 --- a/api/tests/test_check_dem.py +++ b/api/tests/test_check_dem.py @@ -19,9 +19,10 @@ def test_check_dem_coverage(): 'polygon': nsew(51.7, 51.3, 179.7, -179.3), 'name': 'good2', }, - { # not enough coverage of Aleutian Islands over antimeridian; should FAIL with fixed antimeridian + { # not enough coverage of Aleutian Islands over antimeridian + # NOTE: Passes today but should FAIL with antimeridian feature fix 'polygon': nsew(51.7, 41.3, 179.7, -179.3), - 'name': 'shouldbebad1', + 'name': 'good3', }, { # completely encloses tile over Ascension Island in the Atlantic 'polygon': nsew(-6, -9, -15, -14), From e08e788a537b6c5866fabc550c396a11da026743 Mon Sep 17 00:00:00 2001 From: Andrew Johnston Date: Mon, 17 Aug 2020 16:01:58 -0800 Subject: [PATCH 9/9] update changelog for 0.4.0 release --- CHANGELOG.md | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 363f9fed4..8bae4c6f8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,7 +4,10 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). -## [Unreleased] +## [0.4.0] +### Changed +- Implemented 0.15° buffer and 20% threshold in DEM coverage checks when submitting new jobs. As a result slightly more granules will be rejected as having insufficient coverage. + ### Removed - Removed optional `description` field for jobs