From 21bda737b3037493107a23f61d728659afb168da Mon Sep 17 00:00:00 2001 From: Lukas Joss Date: Fri, 9 Aug 2024 15:11:41 +0200 Subject: [PATCH 01/17] PB-200: Add validation check for projection of item --- app/stac_api/admin.py | 2 +- app/stac_api/validators.py | 7 ++++++- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/app/stac_api/admin.py b/app/stac_api/admin.py index 3f0f604d..57d44b12 100644 --- a/app/stac_api/admin.py +++ b/app/stac_api/admin.py @@ -157,7 +157,7 @@ class CollectionFilterForItems(AutocompleteFilter): # helper form to add an extra text_geometry field to ItemAdmin class ItemAdminForm(forms.ModelForm): help_text = """Insert either:
- - An extent in either WGS84 or LV95: "xmin, ymin, xmax, ymax" + - An extent in WGS84: "xmin, ymin, xmax, ymax" where x is easting and y is northing
- A WKT polygon. F.ex. "SRID=4326;POLYGON((5.96 45.82, 5.96 47.81, 10.49 47.81, 10.49 45.82, 5.96 45.82))" diff --git a/app/stac_api/validators.py b/app/stac_api/validators.py index 9689dd53..0cc06b75 100644 --- a/app/stac_api/validators.py +++ b/app/stac_api/validators.py @@ -303,7 +303,7 @@ def validate_text_to_geometry(text_geometry): A validator function that tests, if a text can be transformed to a geometry. The text is either a bbox or WKT. - an extent in either WGS84 or LV95, in the form "(xmin, ymin, xmax, ymax)" where x is easting + an extent in WGS84, in the form "(xmin, ymin, xmax, ymax)" where x is easting a WKT defintion of a polygon in the form POLYGON ((30 10, 40 40, 20 40, 10 20, 30 10)) Args: @@ -361,6 +361,11 @@ def validate_geometry(geometry): params = {'error': geos_geometry.valid_reason} logger.error(message, params) raise ValidationError(_(message), params=params, code='invalid') + if geos_geometry.srid and geos_geometry.srid != 4326: + message = "Only WGS84 projection is permitted (SRID=4326)" + params = {'error': geos_geometry.wkt} + logger.error(message, params) + raise ValidationError(_(message), params=params, code='invalid') return geometry From 1abd8d863dc2665b441ead39d33253a94e0d3823 Mon Sep 17 00:00:00 2001 From: Lukas Joss Date: Mon, 12 Aug 2024 15:38:43 +0200 Subject: [PATCH 02/17] PB-200: Add validation for location of item --- app/stac_api/validators.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/app/stac_api/validators.py b/app/stac_api/validators.py index 0cc06b75..74a5d56b 100644 --- a/app/stac_api/validators.py +++ b/app/stac_api/validators.py @@ -351,6 +351,7 @@ def validate_geometry(geometry): ValidateionError: About that the geometry is not valid ''' geos_geometry = GEOSGeometry(geometry) + max_extent = GEOSGeometry("POLYGON ((3 44,3 50,14 50,14 44,3 44))") if geos_geometry.empty: message = "The geometry is empty: %(error)s" params = {'error': geos_geometry.wkt} @@ -366,6 +367,11 @@ def validate_geometry(geometry): params = {'error': geos_geometry.wkt} logger.error(message, params) raise ValidationError(_(message), params=params, code='invalid') + if not geos_geometry.within(max_extent): + message = "Location of asset outside of permitted region" + params = {'error': geos_geometry.wkt} + logger.error(message, params) + raise ValidationError(_(message), params=params, code='invalid') return geometry From 9e45aea7a17c7b813f29c481bc2223df8616c166 Mon Sep 17 00:00:00 2001 From: Lukas Joss Date: Mon, 12 Aug 2024 16:47:37 +0200 Subject: [PATCH 03/17] PB-200: Add test for projection check --- app/tests/tests_10/test_item_model.py | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/app/tests/tests_10/test_item_model.py b/app/tests/tests_10/test_item_model.py index e90e152d..2150b699 100644 --- a/app/tests/tests_10/test_item_model.py +++ b/app/tests/tests_10/test_item_model.py @@ -168,6 +168,21 @@ def test_item_create_model_invalid_geometry(self): item.full_clean() item.save() + def test_item_create_model_non_standard_geometry(self): + # a geometry not in wgs84 should not be accepted + with self.assertRaises(ValidationError): + item = Item( + collection=self.collection, + properties_datetime=utc_aware(datetime.utcnow()), + name='item-1', + geometry=GEOSGeometry( + 'SRID=1234;POLYGON ' + '((5.96 45.82, 5.96 47.81, 10.49 47.81, 10.49 45.82, 5.96 45.82))' + ) + ) + item.full_clean() + item.save() + def test_item_create_model_empty_geometry(self): # empty geometry should not be allowed with self.assertRaises(ValidationError): From 5fedde605fcc5e6dbea3f431187e4a6e3cc61d88 Mon Sep 17 00:00:00 2001 From: Lukas Joss Date: Tue, 13 Aug 2024 15:56:27 +0200 Subject: [PATCH 04/17] PB-200: Change location error to warning --- app/stac_api/validators.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/app/stac_api/validators.py b/app/stac_api/validators.py index 74a5d56b..0a5c570d 100644 --- a/app/stac_api/validators.py +++ b/app/stac_api/validators.py @@ -368,10 +368,9 @@ def validate_geometry(geometry): logger.error(message, params) raise ValidationError(_(message), params=params, code='invalid') if not geos_geometry.within(max_extent): - message = "Location of asset outside of permitted region" - params = {'error': geos_geometry.wkt} - logger.error(message, params) - raise ValidationError(_(message), params=params, code='invalid') + message = "Location of asset outside of Switzerland" + params = {'warning': geos_geometry.wkt} + logger.warning(message, params) return geometry From ac528ec87a2d9dfb20af92faea9648229915ef0c Mon Sep 17 00:00:00 2001 From: Lukas Joss Date: Tue, 13 Aug 2024 17:48:20 +0200 Subject: [PATCH 05/17] PB-200: Make location check after transformation to wgs84 --- app/stac_api/admin.py | 2 +- app/stac_api/validators.py | 19 +++++++++++++------ 2 files changed, 14 insertions(+), 7 deletions(-) diff --git a/app/stac_api/admin.py b/app/stac_api/admin.py index 57d44b12..3f0f604d 100644 --- a/app/stac_api/admin.py +++ b/app/stac_api/admin.py @@ -157,7 +157,7 @@ class CollectionFilterForItems(AutocompleteFilter): # helper form to add an extra text_geometry field to ItemAdmin class ItemAdminForm(forms.ModelForm): help_text = """Insert either:
- - An extent in WGS84: "xmin, ymin, xmax, ymax" + - An extent in either WGS84 or LV95: "xmin, ymin, xmax, ymax" where x is easting and y is northing
- A WKT polygon. F.ex. "SRID=4326;POLYGON((5.96 45.82, 5.96 47.81, 10.49 47.81, 10.49 45.82, 5.96 45.82))" diff --git a/app/stac_api/validators.py b/app/stac_api/validators.py index 0a5c570d..e3f0c674 100644 --- a/app/stac_api/validators.py +++ b/app/stac_api/validators.py @@ -1,5 +1,6 @@ import logging import re +import sys from collections import namedtuple from datetime import datetime from urllib.parse import urlparse @@ -303,7 +304,7 @@ def validate_text_to_geometry(text_geometry): A validator function that tests, if a text can be transformed to a geometry. The text is either a bbox or WKT. - an extent in WGS84, in the form "(xmin, ymin, xmax, ymax)" where x is easting + an extent in either WGS84 or LV95, in the form "(xmin, ymin, xmax, ymax)" where x is easting a WKT defintion of a polygon in the form POLYGON ((30 10, 40 40, 20 40, 10 20, 30 10)) Args: @@ -317,7 +318,7 @@ def validate_text_to_geometry(text_geometry): # is the input WKT try: geos_geometry = GEOSGeometry(text_geometry) - validate_geometry(geos_geometry) + validate_geometry(geos_geometry, []) return geos_geometry except (ValueError, ValidationError, IndexError, GDALException, GEOSException) as error: message = "The text as WKT could not be transformed into a geometry: %(error)s" @@ -328,7 +329,7 @@ def validate_text_to_geometry(text_geometry): try: text_geometry = text_geometry.replace('(', '') text_geometry = text_geometry.replace(')', '') - return validate_geometry(geometry_from_bbox(text_geometry)) + return validate_geometry(geometry_from_bbox(text_geometry), [2056, 4326]) except (ValueError, ValidationError, IndexError, GDALException) as error: message = "The text as bbox could not be transformed into a geometry: %(error)s" params = {'error': error} @@ -337,7 +338,7 @@ def validate_text_to_geometry(text_geometry): raise ValidationError(errors) from None -def validate_geometry(geometry): +def validate_geometry(geometry, allowed_srid=[4326]): ''' A validator function that ensures, that only valid geometries are stored. @@ -350,6 +351,7 @@ def validate_geometry(geometry): Raises: ValidateionError: About that the geometry is not valid ''' + geos_geometry = GEOSGeometry(geometry) max_extent = GEOSGeometry("POLYGON ((3 44,3 50,14 50,14 44,3 44))") if geos_geometry.empty: @@ -362,8 +364,13 @@ def validate_geometry(geometry): params = {'error': geos_geometry.valid_reason} logger.error(message, params) raise ValidationError(_(message), params=params, code='invalid') - if geos_geometry.srid and geos_geometry.srid != 4326: - message = "Only WGS84 projection is permitted (SRID=4326)" + + if not allowed_srid: + # return geometry if it is a wkt check + return geometry + + if geos_geometry.srid and not geos_geometry.srid in allowed_srid: + message = "Projection is not permitted (allowed projections are SRID="+allowed_srid+")" params = {'error': geos_geometry.wkt} logger.error(message, params) raise ValidationError(_(message), params=params, code='invalid') From ce89250a79b2cf3e21bb16e2b3e56ecdb575db4c Mon Sep 17 00:00:00 2001 From: Lukas Joss Date: Wed, 14 Aug 2024 16:43:28 +0200 Subject: [PATCH 06/17] PB-200: Add explicit geometry transport and lat-lon check --- app/stac_api/utils.py | 2 ++ app/stac_api/validators.py | 38 ++++++++++++++++++++++++++++---------- 2 files changed, 30 insertions(+), 10 deletions(-) diff --git a/app/stac_api/utils.py b/app/stac_api/utils.py index f420c109..af354a17 100644 --- a/app/stac_api/utils.py +++ b/app/stac_api/utils.py @@ -458,6 +458,8 @@ def geometry_from_bbox(bbox): # if large values, SRID is LV95. The default SRID is 4326 if list_bbox_values[0] > 360: bbox_geometry.srid = 2056 + else: + bbox_geometry.srid = 4326 if not bbox_geometry.valid: raise ValueError(f'{bbox_geometry.valid_reason} for bbox with {bbox_geometry.wkt}') diff --git a/app/stac_api/validators.py b/app/stac_api/validators.py index e3f0c674..0d4db782 100644 --- a/app/stac_api/validators.py +++ b/app/stac_api/validators.py @@ -318,7 +318,7 @@ def validate_text_to_geometry(text_geometry): # is the input WKT try: geos_geometry = GEOSGeometry(text_geometry) - validate_geometry(geos_geometry, []) + validate_geometry(geos_geometry, True) return geos_geometry except (ValueError, ValidationError, IndexError, GDALException, GEOSException) as error: message = "The text as WKT could not be transformed into a geometry: %(error)s" @@ -329,7 +329,7 @@ def validate_text_to_geometry(text_geometry): try: text_geometry = text_geometry.replace('(', '') text_geometry = text_geometry.replace(')', '') - return validate_geometry(geometry_from_bbox(text_geometry), [2056, 4326]) + return validate_geometry(geometry_from_bbox(text_geometry), True) except (ValueError, ValidationError, IndexError, GDALException) as error: message = "The text as bbox could not be transformed into a geometry: %(error)s" params = {'error': error} @@ -338,7 +338,7 @@ def validate_text_to_geometry(text_geometry): raise ValidationError(errors) from None -def validate_geometry(geometry, allowed_srid=[4326]): +def validate_geometry(geometry, apply_transform=False): ''' A validator function that ensures, that only valid geometries are stored. @@ -364,16 +364,34 @@ def validate_geometry(geometry, allowed_srid=[4326]): params = {'error': geos_geometry.valid_reason} logger.error(message, params) raise ValidationError(_(message), params=params, code='invalid') + if not geos_geometry.srid: + message = "No projection provided: SRID=%(error)s" + params = {'error': geos_geometry.srid} + logger.error(message, params) + raise ValidationError(_(message), params=params, code='invalid') - if not allowed_srid: - # return geometry if it is a wkt check - return geometry - - if geos_geometry.srid and not geos_geometry.srid in allowed_srid: - message = "Projection is not permitted (allowed projections are SRID="+allowed_srid+")" - params = {'error': geos_geometry.wkt} + # transform geometry from textfield input if necessary + if apply_transform and geos_geometry.srid != 4326: + geos_geometry.transform(4326) + elif geos_geometry.srid != 4326: + message = "Non permitted Projection. Projection must be wgs84 (SRID=4326) instead of SRID=%(error)s" + params = {'error': geos_geometry.srid} logger.error(message, params) raise ValidationError(_(message), params=params, code='invalid') + + if geometry.geom_type == 'Polygon': + for point in geos_geometry[0]: + print(point[0], point[1]) + if abs(point[1]) > 90: + message = "Latitude exceeds permitted value: %(error)s" + params = {'error': point[1]} + logger.error(message, params) + raise ValidationError(_(message), params=params, code='invalid') + if abs(point[0]) > 180: + message = "Longitude exceeds usual value range: %(warning)s" + params = {'warning': point[0]} + logger.warning(message, params) + if not geos_geometry.within(max_extent): message = "Location of asset outside of Switzerland" params = {'warning': geos_geometry.wkt} From 3a01ca554e4806affc94e8c51dbac1c55e8037b2 Mon Sep 17 00:00:00 2001 From: Lukas Joss Date: Wed, 14 Aug 2024 17:44:56 +0200 Subject: [PATCH 07/17] PB-200: Add latitude out of bounds test --- app/stac_api/validators.py | 4 +--- app/tests/tests_09/test_item_model.py | 13 +++++++++++++ app/tests/tests_10/test_item_model.py | 28 +++++++++++++-------------- 3 files changed, 27 insertions(+), 18 deletions(-) diff --git a/app/stac_api/validators.py b/app/stac_api/validators.py index 0d4db782..a40e270a 100644 --- a/app/stac_api/validators.py +++ b/app/stac_api/validators.py @@ -1,6 +1,5 @@ import logging import re -import sys from collections import namedtuple from datetime import datetime from urllib.parse import urlparse @@ -379,9 +378,8 @@ def validate_geometry(geometry, apply_transform=False): logger.error(message, params) raise ValidationError(_(message), params=params, code='invalid') - if geometry.geom_type == 'Polygon': + if geos_geometry.geom_type == 'Polygon': for point in geos_geometry[0]: - print(point[0], point[1]) if abs(point[1]) > 90: message = "Latitude exceeds permitted value: %(error)s" params = {'error': point[1]} diff --git a/app/tests/tests_09/test_item_model.py b/app/tests/tests_09/test_item_model.py index 3e7e5611..0b5d4ceb 100644 --- a/app/tests/tests_09/test_item_model.py +++ b/app/tests/tests_09/test_item_model.py @@ -172,3 +172,16 @@ def test_item_create_model_valid_linestring_geometry(self): ) item.full_clean() item.save() + + def test_item_create_model_invalid_latitude(self): + # a geometry with self-intersection should not be allowed + with self.assertRaises(ValidationError): + item = Item( + collection=self.collection, + properties_datetime=utc_aware(datetime.utcnow()), + name='item-1', + geometry=GEOSGeometry('SRID=4326;POINT (0 100)') + + ) + item.full_clean() + item.save() diff --git a/app/tests/tests_10/test_item_model.py b/app/tests/tests_10/test_item_model.py index 2150b699..da614908 100644 --- a/app/tests/tests_10/test_item_model.py +++ b/app/tests/tests_10/test_item_model.py @@ -168,21 +168,6 @@ def test_item_create_model_invalid_geometry(self): item.full_clean() item.save() - def test_item_create_model_non_standard_geometry(self): - # a geometry not in wgs84 should not be accepted - with self.assertRaises(ValidationError): - item = Item( - collection=self.collection, - properties_datetime=utc_aware(datetime.utcnow()), - name='item-1', - geometry=GEOSGeometry( - 'SRID=1234;POLYGON ' - '((5.96 45.82, 5.96 47.81, 10.49 47.81, 10.49 45.82, 5.96 45.82))' - ) - ) - item.full_clean() - item.save() - def test_item_create_model_empty_geometry(self): # empty geometry should not be allowed with self.assertRaises(ValidationError): @@ -228,3 +213,16 @@ def test_item_create_model_valid_linestring_geometry(self): ) item.full_clean() item.save() + + def test_item_create_model_invalid_latitude(self): + # a geometry with self-intersection should not be allowed + with self.assertRaises(ValidationError): + item = Item( + collection=self.collection, + properties_datetime=utc_aware(datetime.utcnow()), + name='item-1', + geometry=GEOSGeometry('SRID=4326;POINT (0 100)') + + ) + item.full_clean() + item.save() From 015c8954ce94324d9a19e462d203b95bb2556e17 Mon Sep 17 00:00:00 2001 From: Lukas Joss Date: Thu, 15 Aug 2024 11:09:09 +0200 Subject: [PATCH 08/17] PB-200: Update latitude out of bounds check --- app/stac_api/validators.py | 22 +++++++------- app/tests/tests_09/test_item_model.py | 41 ++++++++++++++++++--------- app/tests/tests_10/test_item_model.py | 41 ++++++++++++++++++--------- 3 files changed, 66 insertions(+), 38 deletions(-) diff --git a/app/stac_api/validators.py b/app/stac_api/validators.py index a40e270a..a775f9aa 100644 --- a/app/stac_api/validators.py +++ b/app/stac_api/validators.py @@ -350,7 +350,6 @@ def validate_geometry(geometry, apply_transform=False): Raises: ValidateionError: About that the geometry is not valid ''' - geos_geometry = GEOSGeometry(geometry) max_extent = GEOSGeometry("POLYGON ((3 44,3 50,14 50,14 44,3 44))") if geos_geometry.empty: @@ -378,17 +377,16 @@ def validate_geometry(geometry, apply_transform=False): logger.error(message, params) raise ValidationError(_(message), params=params, code='invalid') - if geos_geometry.geom_type == 'Polygon': - for point in geos_geometry[0]: - if abs(point[1]) > 90: - message = "Latitude exceeds permitted value: %(error)s" - params = {'error': point[1]} - logger.error(message, params) - raise ValidationError(_(message), params=params, code='invalid') - if abs(point[0]) > 180: - message = "Longitude exceeds usual value range: %(warning)s" - params = {'warning': point[0]} - logger.warning(message, params) + extent=geos_geometry.extent + if abs(extent[1]) > 90 or abs(extent[-1]) > 90: + message = "Latitude exceeds permitted value: %(error)s" + params = {'error': (extent[1],extent[-1])} + logger.error(message, params) + raise ValidationError(_(message), params=params, code='invalid') + if abs(extent[0]) > 180 or abs(extent[-2]) > 180: + message = "Longitude exceeds usual value range: %(warning)s" + params = {'warning': (extent[0],extent[-2])} + logger.warning(message, params) if not geos_geometry.within(max_extent): message = "Location of asset outside of Switzerland" diff --git a/app/tests/tests_09/test_item_model.py b/app/tests/tests_09/test_item_model.py index 0b5d4ceb..387bd56c 100644 --- a/app/tests/tests_09/test_item_model.py +++ b/app/tests/tests_09/test_item_model.py @@ -127,6 +127,21 @@ def test_item_create_model_invalid_geometry(self): item.full_clean() item.save() + def test_item_create_model_invalid_latitude(self): + # a geometry with self-intersection should not be allowed + with self.assertRaises(ValidationError): + item = Item( + collection=self.collection, + properties_datetime=utc_aware(datetime.utcnow()), + name='item-1', + geometry=GEOSGeometry( + 'SRID=4326;POLYGON ' + '((5.96 45.82, 5.96 97.81, 10.49 97.81, 10.49 45.82, 5.96 45.82))' + ) + ) + item.full_clean() + item.save() + def test_item_create_model_empty_geometry(self): # empty geometry should not be allowed with self.assertRaises(ValidationError): @@ -162,26 +177,26 @@ def test_item_create_model_valid_point_geometry(self): item.full_clean() item.save() - def test_item_create_model_valid_linestring_geometry(self): - # a correct geometry should not pose any problems - item = Item( - collection=self.collection, - properties_datetime=utc_aware(datetime.utcnow()), - name='item-1', - geometry=GEOSGeometry('SRID=4326;LINESTRING (5.96 45.82, 5.96 47.81)') - ) - item.full_clean() - item.save() - - def test_item_create_model_invalid_latitude(self): + def test_item_create_model_point_geometry_invalid_latitude(self): # a geometry with self-intersection should not be allowed with self.assertRaises(ValidationError): item = Item( collection=self.collection, properties_datetime=utc_aware(datetime.utcnow()), name='item-1', - geometry=GEOSGeometry('SRID=4326;POINT (0 100)') + geometry=GEOSGeometry('SRID=4326;POINT (5.96 95.82)') ) item.full_clean() item.save() + + def test_item_create_model_valid_linestring_geometry(self): + # a correct geometry should not pose any problems + item = Item( + collection=self.collection, + properties_datetime=utc_aware(datetime.utcnow()), + name='item-1', + geometry=GEOSGeometry('SRID=4326;LINESTRING (5.96 45.82, 5.96 47.81)') + ) + item.full_clean() + item.save() diff --git a/app/tests/tests_10/test_item_model.py b/app/tests/tests_10/test_item_model.py index da614908..2a98cffa 100644 --- a/app/tests/tests_10/test_item_model.py +++ b/app/tests/tests_10/test_item_model.py @@ -168,6 +168,21 @@ def test_item_create_model_invalid_geometry(self): item.full_clean() item.save() + def test_item_create_model_invalid_latitude(self): + # a geometry with self-intersection should not be allowed + with self.assertRaises(ValidationError): + item = Item( + collection=self.collection, + properties_datetime=utc_aware(datetime.utcnow()), + name='item-1', + geometry=GEOSGeometry( + 'SRID=4326;POLYGON ' + '((5.96 45.82, 5.96 97.81, 10.49 97.81, 10.49 45.82, 5.96 45.82))' + ) + ) + item.full_clean() + item.save() + def test_item_create_model_empty_geometry(self): # empty geometry should not be allowed with self.assertRaises(ValidationError): @@ -203,26 +218,26 @@ def test_item_create_model_valid_point_geometry(self): item.full_clean() item.save() - def test_item_create_model_valid_linestring_geometry(self): - # a correct geometry should not pose any problems - item = Item( - collection=self.collection, - properties_datetime=utc_aware(datetime.utcnow()), - name='item-1', - geometry=GEOSGeometry('SRID=4326;LINESTRING (5.96 45.82, 5.96 47.81)') - ) - item.full_clean() - item.save() - - def test_item_create_model_invalid_latitude(self): + def test_item_create_model_point_geometry_invalid_latitude(self): # a geometry with self-intersection should not be allowed with self.assertRaises(ValidationError): item = Item( collection=self.collection, properties_datetime=utc_aware(datetime.utcnow()), name='item-1', - geometry=GEOSGeometry('SRID=4326;POINT (0 100)') + geometry=GEOSGeometry('SRID=4326;POINT (5.96 95.82)') ) item.full_clean() item.save() + + def test_item_create_model_valid_linestring_geometry(self): + # a correct geometry should not pose any problems + item = Item( + collection=self.collection, + properties_datetime=utc_aware(datetime.utcnow()), + name='item-1', + geometry=GEOSGeometry('SRID=4326;LINESTRING (5.96 45.82, 5.96 47.81)') + ) + item.full_clean() + item.save() From 8a248b74079a303c9a99b2f25f7c295685a80217 Mon Sep 17 00:00:00 2001 From: Lukas Joss Date: Thu, 15 Aug 2024 11:54:07 +0200 Subject: [PATCH 09/17] PB-200: Add projection test --- app/tests/tests_09/test_item_model.py | 14 ++++++++++++++ app/tests/tests_10/test_item_model.py | 14 ++++++++++++++ 2 files changed, 28 insertions(+) diff --git a/app/tests/tests_09/test_item_model.py b/app/tests/tests_09/test_item_model.py index 387bd56c..05b61f66 100644 --- a/app/tests/tests_09/test_item_model.py +++ b/app/tests/tests_09/test_item_model.py @@ -127,6 +127,20 @@ def test_item_create_model_invalid_geometry(self): item.full_clean() item.save() + def test_item_create_model_invalid_projection(self): + # a geometry with a projection other than wgs84 should not be allowed + with self.assertRaises(ValidationError): + item = Item( + collection=self.collection, + properties_datetime=utc_aware(datetime.utcnow()), + name='item-1', + geometry=GEOSGeometry( + 'SRID=2056;POLYGON ((2500000 1100000, 2600000 1100000, 2600000 1200000, 2500000 1200000, 2500000 1100000))' + ) + ) + item.full_clean() + item.save() + def test_item_create_model_invalid_latitude(self): # a geometry with self-intersection should not be allowed with self.assertRaises(ValidationError): diff --git a/app/tests/tests_10/test_item_model.py b/app/tests/tests_10/test_item_model.py index 2a98cffa..134dfa01 100644 --- a/app/tests/tests_10/test_item_model.py +++ b/app/tests/tests_10/test_item_model.py @@ -168,6 +168,20 @@ def test_item_create_model_invalid_geometry(self): item.full_clean() item.save() + def test_item_create_model_invalid_projection(self): + # a geometry with a projection other than wgs84 should not be allowed + with self.assertRaises(ValidationError): + item = Item( + collection=self.collection, + properties_datetime=utc_aware(datetime.utcnow()), + name='item-1', + geometry=GEOSGeometry( + 'SRID=2056;POLYGON ((2500000 1100000, 2600000 1100000, 2600000 1200000, 2500000 1200000, 2500000 1100000))' + ) + ) + item.full_clean() + item.save() + def test_item_create_model_invalid_latitude(self): # a geometry with self-intersection should not be allowed with self.assertRaises(ValidationError): From c214d1a32108d85ca34de5ddb6b069983320d9fe Mon Sep 17 00:00:00 2001 From: Lukas Joss Date: Thu, 15 Aug 2024 14:30:30 +0200 Subject: [PATCH 10/17] PB-200: Add test that geometries in admin console are properly converted --- app/stac_api/validators.py | 19 +++++----- app/tests/test_admin_page.py | 51 +++++++++++++++++++-------- app/tests/tests_09/test_item_model.py | 4 +-- app/tests/tests_10/test_item_model.py | 4 +-- 4 files changed, 51 insertions(+), 27 deletions(-) diff --git a/app/stac_api/validators.py b/app/stac_api/validators.py index a775f9aa..ab2ecfd7 100644 --- a/app/stac_api/validators.py +++ b/app/stac_api/validators.py @@ -317,7 +317,7 @@ def validate_text_to_geometry(text_geometry): # is the input WKT try: geos_geometry = GEOSGeometry(text_geometry) - validate_geometry(geos_geometry, True) + validate_geometry(geos_geometry, apply_transform=True) return geos_geometry except (ValueError, ValidationError, IndexError, GDALException, GEOSException) as error: message = "The text as WKT could not be transformed into a geometry: %(error)s" @@ -328,7 +328,7 @@ def validate_text_to_geometry(text_geometry): try: text_geometry = text_geometry.replace('(', '') text_geometry = text_geometry.replace(')', '') - return validate_geometry(geometry_from_bbox(text_geometry), True) + return validate_geometry(geometry_from_bbox(text_geometry), apply_transform=True) except (ValueError, ValidationError, IndexError, GDALException) as error: message = "The text as bbox could not be transformed into a geometry: %(error)s" params = {'error': error} @@ -351,7 +351,7 @@ def validate_geometry(geometry, apply_transform=False): ValidateionError: About that the geometry is not valid ''' geos_geometry = GEOSGeometry(geometry) - max_extent = GEOSGeometry("POLYGON ((3 44,3 50,14 50,14 44,3 44))") + bbox_ch = GEOSGeometry("POLYGON ((3 44,3 50,14 50,14 44,3 44))") if geos_geometry.empty: message = "The geometry is empty: %(error)s" params = {'error': geos_geometry.wkt} @@ -372,24 +372,25 @@ def validate_geometry(geometry, apply_transform=False): if apply_transform and geos_geometry.srid != 4326: geos_geometry.transform(4326) elif geos_geometry.srid != 4326: - message = "Non permitted Projection. Projection must be wgs84 (SRID=4326) instead of SRID=%(error)s" + message = 'Non permitted Projection. Projection must be wgs84 (SRID=4326) instead of ' \ + 'SRID=%(error)s' params = {'error': geos_geometry.srid} logger.error(message, params) raise ValidationError(_(message), params=params, code='invalid') - extent=geos_geometry.extent + extent = geos_geometry.extent if abs(extent[1]) > 90 or abs(extent[-1]) > 90: message = "Latitude exceeds permitted value: %(error)s" - params = {'error': (extent[1],extent[-1])} + params = {'error': (extent[1], extent[-1])} logger.error(message, params) raise ValidationError(_(message), params=params, code='invalid') if abs(extent[0]) > 180 or abs(extent[-2]) > 180: message = "Longitude exceeds usual value range: %(warning)s" - params = {'warning': (extent[0],extent[-2])} + params = {'warning': (extent[0], extent[-2])} logger.warning(message, params) - if not geos_geometry.within(max_extent): - message = "Location of asset outside of Switzerland" + if not geos_geometry.within(bbox_ch): + message = "Location of asset is (partially) outside of Switzerland" params = {'warning': geos_geometry.wkt} logger.warning(message, params) return geometry diff --git a/app/tests/test_admin_page.py b/app/tests/test_admin_page.py index a0e41a92..3d66fe38 100644 --- a/app/tests/test_admin_page.py +++ b/app/tests/test_admin_page.py @@ -118,24 +118,27 @@ def _create_collection( return collection, data, link, provider - def _create_item(self, collection, with_link=False, extra=None): + def _create_item(self, collection, with_link=False, extra=None, data=None): # Post data to create a new item # Note: the *-*_FORMS fields are necessary management form fields # originating from the AdminInline and must be present - data = { - "collection": collection.id, - "name": "test_item", - "geometry": - "SRID=4326;POLYGON((5.96 45.82, 5.96 47.81, 10.49 47.81, 10.49 45.82, 5.96 45.82))", - "text_geometry": - "SRID=4326;POLYGON((5.96 45.82, 5.96 47.81, 10.49 47.81, 10.49 45.82, 5.96 45.82))", - "properties_datetime_0": "2020-12-01", - "properties_datetime_1": "13:15:39", - "properties_title": "test", - "links-TOTAL_FORMS": "0", - "links-INITIAL_FORMS": "0", - } + if not data: + data = { + "collection": collection.id, + "name": "test_item", + "geometry": + "SRID=4326;POLYGON((5.96 45.82, 5.96 47.81, 10.49 47.81, 10.49 45.82, "\ + "5.96 45.82))", + "text_geometry": + "SRID=4326;POLYGON((5.96 45.82, 5.96 47.81, 10.49 47.81, 10.49 45.82, "\ + "5.96 45.82))", + "properties_datetime_0": "2020-12-01", + "properties_datetime_1": "13:15:39", + "properties_title": "test", + "links-TOTAL_FORMS": "0", + "links-INITIAL_FORMS": "0", + } if with_link: data.update({ "links-TOTAL_FORMS": "1", @@ -672,6 +675,26 @@ def test_add_update_item(self): msg="Admin page item properties_title update did not work" ) + def test_add_item_with_non_standard_projection(self): + geometry = "SRID=4326;POLYGON ((6.146799690987942 46.04410910398307, "\ + "7.438647976247294 46.05153158188484, 7.438632420871813 46.951082771871064, "\ + "6.125143650928986 46.94353699772178, 6.146799690987942 46.04410910398307))" + text_geometry = "SRID=2056;POLYGON ((2500000 1100000, 2600000 1100000, 2600000 1200000, "\ + "2500000 1200000, 2500000 1100000))" + post_data = { + "collection": self.collection.id, + "name": "test_item", + "geometry": geometry, + "text_geometry": text_geometry, + "properties_datetime_0": "2020-12-01", + "properties_datetime_1": "13:15:39", + "properties_title": "test", + "links-TOTAL_FORMS": "0", + "links-INITIAL_FORMS": "0", + } + #if transformed text_geometry does not match the geometry provided the creation will fail + self._create_item(self.collection, data=post_data)[:2] # pylint: disable=expression-not-assigned + def test_add_update_item_remove_title(self): item, data = self._create_item(self.collection)[:2] diff --git a/app/tests/tests_09/test_item_model.py b/app/tests/tests_09/test_item_model.py index 05b61f66..b7c8cfd5 100644 --- a/app/tests/tests_09/test_item_model.py +++ b/app/tests/tests_09/test_item_model.py @@ -135,7 +135,8 @@ def test_item_create_model_invalid_projection(self): properties_datetime=utc_aware(datetime.utcnow()), name='item-1', geometry=GEOSGeometry( - 'SRID=2056;POLYGON ((2500000 1100000, 2600000 1100000, 2600000 1200000, 2500000 1200000, 2500000 1100000))' + 'SRID=2056;POLYGON ((2500000 1100000, 2600000 1100000, 2600000 1200000, ' \ + '2500000 1200000, 2500000 1100000))' ) ) item.full_clean() @@ -199,7 +200,6 @@ def test_item_create_model_point_geometry_invalid_latitude(self): properties_datetime=utc_aware(datetime.utcnow()), name='item-1', geometry=GEOSGeometry('SRID=4326;POINT (5.96 95.82)') - ) item.full_clean() item.save() diff --git a/app/tests/tests_10/test_item_model.py b/app/tests/tests_10/test_item_model.py index 134dfa01..2c61789f 100644 --- a/app/tests/tests_10/test_item_model.py +++ b/app/tests/tests_10/test_item_model.py @@ -176,7 +176,8 @@ def test_item_create_model_invalid_projection(self): properties_datetime=utc_aware(datetime.utcnow()), name='item-1', geometry=GEOSGeometry( - 'SRID=2056;POLYGON ((2500000 1100000, 2600000 1100000, 2600000 1200000, 2500000 1200000, 2500000 1100000))' + 'SRID=2056;POLYGON ((2500000 1100000, 2600000 1100000, 2600000 1200000, ' \ + '2500000 1200000, 2500000 1100000))' ) ) item.full_clean() @@ -240,7 +241,6 @@ def test_item_create_model_point_geometry_invalid_latitude(self): properties_datetime=utc_aware(datetime.utcnow()), name='item-1', geometry=GEOSGeometry('SRID=4326;POINT (5.96 95.82)') - ) item.full_clean() item.save() From bc38d2985070f1d88acdb308594a1e996581e171 Mon Sep 17 00:00:00 2001 From: Benjamin Sugden Date: Thu, 15 Aug 2024 11:08:25 +0200 Subject: [PATCH 11/17] Add console-address port for minio server This allows to correctly use the minio browser. --- docker-compose.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/docker-compose.yml b/docker-compose.yml index d9bef2ac..b12c449f 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -18,13 +18,14 @@ services: image: minio/minio env_file: ./minio.env user: ${UID} - command: server /data + command: server /data --console-address ":9001" volumes: - type: bind source: ${PWD}/.volumes/minio target: /data ports: - 9090:${S3_PORT:-9000} + - 9001:9001 s3-client: image: minio/mc links: From 820f29a3a6f3a270f41852efb67fe0f68342f4b9 Mon Sep 17 00:00:00 2001 From: Benjamin Sugden Date: Thu, 15 Aug 2024 11:11:48 +0200 Subject: [PATCH 12/17] PB-848: Management command to delete expired items Expired items and their assets need to be removed. New management command can be scheduled as cron job to remove expired items. --- app/config/settings_prod.py | 2 + .../commands/remove_expired_items.py | 75 +++++++++++ .../tests_10/test_remove_expired_items.py | 121 ++++++++++++++++++ 3 files changed, 198 insertions(+) create mode 100644 app/stac_api/management/commands/remove_expired_items.py create mode 100644 app/tests/tests_10/test_remove_expired_items.py diff --git a/app/config/settings_prod.py b/app/config/settings_prod.py index c2590b0d..b9eee664 100644 --- a/app/config/settings_prod.py +++ b/app/config/settings_prod.py @@ -192,6 +192,8 @@ '.yml': 'application/vnd.oai.openapi+yaml;version=3.0' } +DELETE_EXPIRED_ITEMS_OLDER_THAN_HOURS = 24 + # Media files (i.e. uploaded content=assets in this project) UPLOAD_FILE_CHUNK_SIZE = 1024 * 1024 # Size in Bytes STORAGES = { diff --git a/app/stac_api/management/commands/remove_expired_items.py b/app/stac_api/management/commands/remove_expired_items.py new file mode 100644 index 00000000..f6fd4cb8 --- /dev/null +++ b/app/stac_api/management/commands/remove_expired_items.py @@ -0,0 +1,75 @@ +from datetime import timedelta + +from django.conf import settings +from django.core.management.base import CommandParser +from django.utils import timezone + +from stac_api.models import Asset +from stac_api.models import Item +from stac_api.utils import CommandHandler +from stac_api.utils import CustomBaseCommand + + +def boolean_input(question, default=None): + result = input(f"{question}") + if not result and default is not None: + return default + return len(result) > 0 and result[0].lower() == "y" + + +class Handler(CommandHandler): + + def delete(self, instance, object_type): + if self.options['dry_run']: + self.print_success(f'skipping deletion of {object_type} {instance}') + else: + instance.delete() + + def run(self): + # print(self.options) + self.print_success('running command to remove expired items') + min_age_hours = self.options['min_age_hours'] + self.print_warning(f"deleting all items expired longer than {min_age_hours} hours") + items = Item.objects.filter( + properties_expires__lte=timezone.now() - timedelta(hours=min_age_hours) + ).all() + for i in items: + assets = Asset.objects.filter(item_id=i.id).all() + assets_length = len(assets) + self.delete(assets, 'assets') + self.delete(i, 'item') + if not self.options['dry_run']: + self.print_success( + f"deleted item {i.name} and {assets_length}" + " assets belonging to it.", + extra={"item": i.name} + ) + + if self.options['dry_run']: + self.print_success(f'[dry run] would have removed {len(items)} expired items') + else: + self.print_success(f'successfully removed {len(items)} expired items') + + +class Command(CustomBaseCommand): + help = """Remove items and their assets that have expired more than + DELETE_EXPIRED_ITEMS_OLDER_THAN_HOURS hours ago. + This command is thought to be scheduled as cron job. + """ + + def add_arguments(self, parser: CommandParser) -> None: + super().add_arguments(parser) + parser.add_argument( + '--dry-run', + action='store_true', + help='Simulate deleting items, without actually deleting them' + ) + default_min_age = settings.DELETE_EXPIRED_ITEMS_OLDER_THAN_HOURS + parser.add_argument( + '--min-age-hours', + type=int, + default=default_min_age, + help=f"Minimum hours the item must have been expired for (default {default_min_age})" + ) + + def handle(self, *args, **options): + Handler(self, options).run() diff --git a/app/tests/tests_10/test_remove_expired_items.py b/app/tests/tests_10/test_remove_expired_items.py new file mode 100644 index 00000000..505dcd33 --- /dev/null +++ b/app/tests/tests_10/test_remove_expired_items.py @@ -0,0 +1,121 @@ +from datetime import timedelta +from io import StringIO + +from django.core.management import call_command +from django.test import TestCase +from django.utils import timezone + +from stac_api.models import Asset +from stac_api.models import Item + +from tests.tests_10.data_factory import Factory +from tests.utils import mock_s3_asset_file + + +class RemoveExpiredItems(TestCase): + + @classmethod + def setUpTestData(cls): + cls.factory = Factory() + cls.collection = cls.factory.create_collection_sample().model + + def _call_command(self, *args, **kwargs): + out = StringIO() + call_command( + "remove_expired_items", + *args, + stdout=out, + stderr=StringIO(), + **kwargs, + ) + return out.getvalue() + + @mock_s3_asset_file + def test_remove_item_dry_run(self): + item_0 = self.factory.create_item_sample( + self.collection, + name='item-0', + db_create=True, + properties_expires=timezone.now() - timedelta(hours=50) + ) + assets = self.factory.create_asset_samples( + 2, item_0.model, name=['asset-0.tiff', 'asset-1.tiff'], db_create=True + ) + + out = self._call_command("--dry-run", "--no-color") + self.assertEqual( + out, + """running command to remove expired items +deleting all items expired longer than 24 hours +skipping deletion of assets , ]> +skipping deletion of item collection-1/item-0 +[dry run] would have removed 1 expired items +""" + ) + + self.assertTrue( + Item.objects.filter(name=item_0['name']).exists(), + msg="Item has been deleted by dry run" + ) + self.assertTrue( + Asset.objects.filter(name=assets[0]['name']).exists(), + msg="Asset has been deleted by dry run" + ) + self.assertTrue( + Asset.objects.filter(name=assets[1]['name']).exists(), + msg="Asset has been deleted by dry run" + ) + + @mock_s3_asset_file + def test_remove_item(self): + item_1 = self.factory.create_item_sample( + self.collection, + name='item-1', + db_create=True, + properties_expires=timezone.now() - timedelta(hours=10) + ) + assets = self.factory.create_asset_samples( + 2, item_1.model, name=['asset-2.tiff', 'asset-3.tiff'], db_create=True + ) + out = self._call_command("--no-color") + self.assertEqual( + out, + """running command to remove expired items +deleting all items expired longer than 24 hours +successfully removed 0 expired items +""" + ) + + self.assertTrue( + Item.objects.filter(name=item_1['name']).exists(), + msg="not expired item has been deleted" + ) + self.assertTrue( + Asset.objects.filter(name=assets[0]['name']).exists(), + msg="not expired asset has been deleted" + ) + self.assertTrue( + Asset.objects.filter(name=assets[1]['name']).exists(), + msg="not expired asset has been deleted" + ) + + out = self._call_command("--min-age-hours=9", "--no-color") + self.assertEqual( + out, + """running command to remove expired items +deleting all items expired longer than 9 hours +deleted item item-1 and 2 assets belonging to it. extra={'item': 'item-1'} +successfully removed 1 expired items +""" + ) + self.assertFalse( + Item.objects.filter(name=item_1['name']).exists(), msg="Expired item was not deleted" + ) + self.assertFalse( + Asset.objects.filter(name=assets[0]['name']).exists(), + msg="Asset of expired item was not deleted" + ) + self.assertFalse( + Asset.objects.filter(name=assets[1]['name']).exists(), + msg="Asset of expired item was not deleted" + ) From cf3f3e4c69e96fbe0eb95deda67a8a3dc2d8ca1e Mon Sep 17 00:00:00 2001 From: Benjamin Sugden Date: Thu, 15 Aug 2024 15:26:31 +0200 Subject: [PATCH 13/17] PB-848: Don't return expired items Remove expired items from list/get items endpoints. Don't return asset details of expired items. Asset files may still be downloaded from s3 until the item is deleted. --- .../commands/remove_expired_items.py | 7 ----- app/stac_api/validators_view.py | 8 +++-- app/stac_api/views.py | 7 +++++ app/tests/tests_10/test_assets_endpoint.py | 29 +++++++++++++++++++ app/tests/tests_10/test_items_endpoint.py | 23 +++++++++++++++ 5 files changed, 65 insertions(+), 9 deletions(-) diff --git a/app/stac_api/management/commands/remove_expired_items.py b/app/stac_api/management/commands/remove_expired_items.py index f6fd4cb8..05f6347d 100644 --- a/app/stac_api/management/commands/remove_expired_items.py +++ b/app/stac_api/management/commands/remove_expired_items.py @@ -10,13 +10,6 @@ from stac_api.utils import CustomBaseCommand -def boolean_input(question, default=None): - result = input(f"{question}") - if not result and default is not None: - return default - return len(result) > 0 and result[0].lower() == "y" - - class Handler(CommandHandler): def delete(self, instance, object_type): diff --git a/app/stac_api/validators_view.py b/app/stac_api/validators_view.py index 5f290dcf..1200f879 100644 --- a/app/stac_api/validators_view.py +++ b/app/stac_api/validators_view.py @@ -1,6 +1,8 @@ import logging +from django.db.models import Q from django.http import Http404 +from django.utils import timezone from django.utils.translation import gettext_lazy as _ from rest_framework import serializers @@ -28,7 +30,7 @@ def validate_collection(kwargs): def validate_item(kwargs): - '''Validate that the item given in request kwargs exists + '''Validate that the item given in request kwargs exists and is not expired Args: kwargs: dict @@ -38,7 +40,9 @@ def validate_item(kwargs): Http404: when the item doesn't exists ''' if not Item.objects.filter( - name=kwargs['item_name'], collection__name=kwargs['collection_name'] + Q(properties_expires=None) | Q(properties_expires__gte=timezone.now()), + name=kwargs['item_name'], + collection__name=kwargs['collection_name'] ).exists(): logger.error( "The item %s is not part of the collection %s", diff --git a/app/stac_api/views.py b/app/stac_api/views.py index 74525ce9..2d75d099 100644 --- a/app/stac_api/views.py +++ b/app/stac_api/views.py @@ -8,6 +8,8 @@ from django.db import transaction from django.db.models import Min from django.db.models import Prefetch +from django.db.models import Q +from django.utils import timezone from django.utils.translation import gettext_lazy as _ from rest_framework import generics @@ -364,6 +366,8 @@ class ItemsList(generics.GenericAPIView): def get_queryset(self): # filter based on the url queryset = Item.objects.filter( + # filter expired items + Q(properties_expires__gte=timezone.now()) | Q(properties_expires=None), collection__name=self.kwargs['collection_name'] ).prefetch_related(Prefetch('assets', queryset=Asset.objects.order_by('name')), 'links') bbox = self.request.query_params.get('bbox', None) @@ -428,6 +432,8 @@ class ItemDetail( def get_queryset(self): # filter based on the url queryset = Item.objects.filter( + # filter expired items + Q(properties_expires__gte=timezone.now()) | Q(properties_expires=None), collection__name=self.kwargs['collection_name'] ).prefetch_related(Prefetch('assets', queryset=Asset.objects.order_by('name')), 'links') @@ -536,6 +542,7 @@ class AssetDetail( def get_queryset(self): # filter based on the url return Asset.objects.filter( + Q(item__properties_expires=None) | Q(item__properties_expires__gte=timezone.now()), item__collection__name=self.kwargs['collection_name'], item__name=self.kwargs['item_name'] ) diff --git a/app/tests/tests_10/test_assets_endpoint.py b/app/tests/tests_10/test_assets_endpoint.py index a56c5577..d2afbd36 100644 --- a/app/tests/tests_10/test_assets_endpoint.py +++ b/app/tests/tests_10/test_assets_endpoint.py @@ -1,5 +1,6 @@ import logging from datetime import datetime +from datetime import timedelta from json import dumps from json import loads from pprint import pformat @@ -7,6 +8,7 @@ from django.contrib.auth import get_user_model from django.test import Client from django.urls import reverse +from django.utils import timezone from stac_api.models import Asset from stac_api.utils import get_asset_path @@ -88,6 +90,19 @@ def test_assets_endpoint_item_does_not_exist(self): ) self.assertStatusCode(404, response) + def test_assets_endpoint_item_expired(self): + collection_name = self.collection.name + item_expired = self.factory.create_item_sample( + self.collection, + name='item-expired', + db_create=True, + properties_expires=timezone.now() - timedelta(hours=1) + ).model + response = self.client.get( + f"/{STAC_BASE_V}/collections/{collection_name}/items/{item_expired.name}/assets" + ) + self.assertStatusCode(404, response) + def test_single_asset_endpoint(self): collection_name = self.collection.name item_name = self.item.name @@ -105,6 +120,20 @@ def test_single_asset_endpoint(self): # hash computation of the ETag self.assertEtagHeader(None, response) + def test_single_assets_endpoint_item_expired(self): + collection_name = self.collection.name + item = self.factory.create_item_sample( + self.collection, + name='item-expired', + db_create=True, + properties_expires=timezone.now() - timedelta(hours=1) + ).model + asset = self.factory.create_asset_sample(item=item, db_create=True).model + response = self.client.get( + f"/{STAC_BASE_V}/collections/{collection_name}/items/{item.name}/assets/{asset.name}" + ) + self.assertStatusCode(404, response) + class AssetsUnimplementedEndpointTestCase(StacBaseTestCase): diff --git a/app/tests/tests_10/test_items_endpoint.py b/app/tests/tests_10/test_items_endpoint.py index 45e3128c..c0761c27 100644 --- a/app/tests/tests_10/test_items_endpoint.py +++ b/app/tests/tests_10/test_items_endpoint.py @@ -5,6 +5,7 @@ from django.contrib.auth import get_user_model from django.test import Client from django.urls import reverse +from django.utils import timezone from stac_api.models import Item from stac_api.utils import fromisoformat @@ -44,6 +45,13 @@ def test_items_endpoint(self): # To make sure that item sorting is working, make sure that the items where not # created in ascending order, same for assets item_3 = self.factory.create_item_sample(self.collection, name='item-0', db_create=True) + # created item that is expired should not show up in the get result + self.factory.create_item_sample( + self.collection, + name='item-expired', + db_create=True, + properties_expires=timezone.now() - timedelta(hours=1) + ) assets = self.factory.create_asset_samples( 3, item_3.model, name=['asset-1.tiff', 'asset-0.tiff', 'asset-2.tiff'], db_create=True ) @@ -146,6 +154,21 @@ def test_single_item_endpoint(self): ignore=['id', 'links'] ) + def test_single_item_endpoint_expired(self): + collection_name = self.collection.name + # created item that is expired should not be found + item = self.factory.create_item_sample( + self.collection, + name='item-expired', + db_create=True, + properties_expires=timezone.now() - timedelta(hours=1) + ) + + response = self.client.get( + f"/{STAC_BASE_V}/collections/{collection_name}/items/{item['name']}" + ) + self.assertStatusCode(404, response) + def test_items_endpoint_non_existing_collection(self): response = self.client.get(f"/{STAC_BASE_V}/collections/non-existing-collection/items") self.assertStatusCode(404, response) From a3a6e64177ba173bce4acbce39a128022e962568 Mon Sep 17 00:00:00 2001 From: Benjamin Sugden Date: Fri, 16 Aug 2024 10:35:47 +0200 Subject: [PATCH 14/17] PB-848: small cleanup after review --- .../management/commands/remove_expired_items.py | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/app/stac_api/management/commands/remove_expired_items.py b/app/stac_api/management/commands/remove_expired_items.py index 05f6347d..86f49409 100644 --- a/app/stac_api/management/commands/remove_expired_items.py +++ b/app/stac_api/management/commands/remove_expired_items.py @@ -4,7 +4,6 @@ from django.core.management.base import CommandParser from django.utils import timezone -from stac_api.models import Asset from stac_api.models import Item from stac_api.utils import CommandHandler from stac_api.utils import CustomBaseCommand @@ -19,22 +18,21 @@ def delete(self, instance, object_type): instance.delete() def run(self): - # print(self.options) self.print_success('running command to remove expired items') min_age_hours = self.options['min_age_hours'] self.print_warning(f"deleting all items expired longer than {min_age_hours} hours") items = Item.objects.filter( properties_expires__lte=timezone.now() - timedelta(hours=min_age_hours) ).all() - for i in items: - assets = Asset.objects.filter(item_id=i.id).all() + for item in items: + assets = item.assets.all() assets_length = len(assets) self.delete(assets, 'assets') - self.delete(i, 'item') + self.delete(item, 'item') if not self.options['dry_run']: self.print_success( - f"deleted item {i.name} and {assets_length}" + " assets belonging to it.", - extra={"item": i.name} + f"deleted item {item.name} and {assets_length}" + " assets belonging to it.", + extra={"item": item.name} ) if self.options['dry_run']: From 62f3938ab903a1cd0806d3fb1142b14318cd2dac Mon Sep 17 00:00:00 2001 From: Benjamin Sugden Date: Wed, 4 Sep 2024 09:08:56 +0200 Subject: [PATCH 15/17] PB-511: Add request context to all logs Add middleware and logging filter to add the django request to all log records. --- Pipfile | 2 +- Pipfile.lock | 8 ++++---- app/config/logging-cfg-local.yml | 6 ++++++ app/config/settings_prod.py | 3 +++ 4 files changed, 14 insertions(+), 5 deletions(-) diff --git a/Pipfile b/Pipfile index c046c7ee..1fd87661 100644 --- a/Pipfile +++ b/Pipfile @@ -43,7 +43,7 @@ py-multihash = "~=2.0.1" django-prometheus = "~=2.3.1" django-admin-autocomplete-filter = "~=0.7.1" django-pgtrigger = "~=4.11.1" -logging-utilities = "~=4.4.1" +logging-utilities = "~=4.5.0" django-environ = "*" [requires] diff --git a/Pipfile.lock b/Pipfile.lock index f067703b..f4d53be7 100644 --- a/Pipfile.lock +++ b/Pipfile.lock @@ -1,7 +1,7 @@ { "_meta": { "hash": { - "sha256": "32b93ecc78a501f105c65882083f8c63a374dc1d70c95ecc8ae4c501337d405d" + "sha256": "26f4b2b9d09c47972e02c969bd13d98ec09f34125905c9231d496e8143b9495a" }, "pipfile-spec": 6, "requires": { @@ -369,12 +369,12 @@ }, "logging-utilities": { "hashes": [ - "sha256:0e144647749fdb153a43cdede0b077b9bcfd50fc6a068e07716b1a002173e8ef", - "sha256:ed3bb7aa657315b08415a92499f5ce38c024cc1aa18b682f1fd55c72b54491b9" + "sha256:0f88c3ddf33a7e81da20839667a9bedadfe22de6f9087c801aa1f582b89b4c93", + "sha256:d96b6910d0a1c26216bca893c11d66c0a8a61113af44cd908b2e4f4ae569e067" ], "index": "pypi", "markers": "python_version >= '3.0'", - "version": "==4.4.1" + "version": "==4.5.0" }, "morphys": { "hashes": [ diff --git a/app/config/logging-cfg-local.yml b/app/config/logging-cfg-local.yml index 430cc884..140d1da5 100644 --- a/app/config/logging-cfg-local.yml +++ b/app/config/logging-cfg-local.yml @@ -38,6 +38,11 @@ filters: (): logging_utilities.filters.TimeAttribute isotime: False utc_isotime: True + add_request: + (): logging_utilities.filters.add_thread_context_filter.AddThreadContextFilter + contexts: + - logger_key: request + context_key: request django: (): logging_utilities.filters.django_request.JsonDjangoRequest attr_name: request @@ -133,6 +138,7 @@ handlers: # handler, they will affect every handler - type_filter - isotime + - add_request - django # This filter only applies to the current handler (It does not modify the record in-place, but # instead selects which logs to display) diff --git a/app/config/settings_prod.py b/app/config/settings_prod.py index b9eee664..8aa6b443 100644 --- a/app/config/settings_prod.py +++ b/app/config/settings_prod.py @@ -81,6 +81,9 @@ # last, put everything else in between MIDDLEWARE = [ 'django_prometheus.middleware.PrometheusBeforeMiddleware', + # Middleware to add request to thread variables, this should be far up in the chain so request + # information can be added to as many logs as possible. + 'logging_utilities.django_middlewares.add_request_context.AddToThreadContextMiddleware', 'middleware.logging.RequestResponseLoggingMiddleware', 'django.middleware.security.SecurityMiddleware', 'middleware.cors.CORSHeadersMiddleware', From 311cb5fa837f7ab1a5f8e28b66adb71506942970 Mon Sep 17 00:00:00 2001 From: Benjamin Sugden Date: Wed, 4 Sep 2024 12:01:50 +0200 Subject: [PATCH 16/17] PB-755: Spec for collection assets Add CRUD endpoints for collection level assets. Add endpoints for collection asset upload management. As process for upload is the same as for feature assets, only link to description. --- spec/static/spec/v1/openapitransactional.yaml | 450 +++++++++++++++++- spec/transaction/components/responses.yaml | 19 +- spec/transaction/paths.yaml | 432 ++++++++++++++++- spec/transaction/tags.yaml | 13 +- 4 files changed, 874 insertions(+), 40 deletions(-) diff --git a/spec/static/spec/v1/openapitransactional.yaml b/spec/static/spec/v1/openapitransactional.yaml index d0e890c2..1259e1b3 100644 --- a/spec/static/spec/v1/openapitransactional.yaml +++ b/spec/static/spec/v1/openapitransactional.yaml @@ -112,7 +112,7 @@ tags: (1) Use the [create new upload](#tag/Asset-Upload-Management/operation/createAssetUpload) request to start a new upload. It will return a list of urls. - (2) Use the urls to [upload asset file part](#tag/Asset-Upload-Management/operation/uploadAssetFilePart). Do this for each file part. + (2) Use the urls to [upload asset file part](#tag/Asset-Upload-Management/operation/uploadAssetFilePart). Do this for each file part. You may also upload multiple parts in parallel. (3) Once all parts have been uploaded, execute the [complete the upload](#tag/Asset-Upload-Management/operation/completeMultipartUpload) request. The new asset file be available to users after you have successfully completed the upload. @@ -296,6 +296,16 @@ tags: ``` See https://aws.amazon.com/premiumsupport/knowledge-center/data-integrity-s3/ for other examples on how to compute the base64 MD5 of a part. + - name: Collection Asset Upload Management + description: | + Collection Asset files are uploaded via the STAC API using the API requests described in this chapter. + + The flow of the requests is the same as for assets that belong to features, which is described under [Asset Upload Management](#tag/Asset-Upload-Management) + - [Basic steps to upload assets](#section/Basic-Steps-to-upload-assets) + - [Detailed example of asset upload](#section/Detailed-example-of-asset-upload) + - [Authentication](#section/Authentication) + - [Compression](#section/Compression) + - [Example](#section/Example) - name: Authentication description: | All write requests require authentication. There is currently three type of supported authentications: @@ -416,7 +426,7 @@ paths: - Data Management summary: Update or create a collection description: >- - Update or create a collection with Id `collectionId` with a complete collection definition. If the collection doesn't exists it is then created. + Update or create a collection with Id `collectionId` with a complete collection definition. If the collection doesn't exist it is created. operationId: updateCollection parameters: - $ref: "#/components/parameters/collectionId" @@ -574,7 +584,7 @@ paths: put: summary: Update or create a feature description: >- - Update or create a feature with Id `featureId` with a complete feature definition. If the feature doesn't exists it is then created. + Update or create a feature with Id `featureId` with a complete feature definition. If the feature doesn't exist it is then created. *NOTE: Optional fields that are not part of the PUT payload, will be erased in the resource. For example if the resource has a properties.title and the PUT payload doesn't, then the resource's properties.title will be removed.* @@ -641,12 +651,12 @@ paths: $ref: "#/components/responses/NotFound" "412": $ref: "#/components/responses/PreconditionFailed" - 5XX: + "500": $ref: "#/components/responses/ServerError" patch: summary: Update an existing feature by Id with a partial item definition description: >- - Use this method to update an existing feature. Requires a JSON fragment (containing the fields to be updated) be submitted. + Use this method to update an existing feature. Requires a JSON fragment (containing the fields to be updated) to be submitted. operationId: patchFeature tags: - Data Management @@ -686,7 +696,7 @@ paths: $ref: "#/components/responses/NotFound" "412": $ref: "#/components/responses/PreconditionFailed" - 5XX: + "500": $ref: "#/components/responses/ServerError" delete: summary: Delete an existing feature by Id @@ -709,7 +719,7 @@ paths: $ref: "#/components/responses/NotFound" "412": $ref: "#/components/responses/PreconditionFailed" - 5XX: + "500": $ref: "#/components/responses/ServerError" /{assetObjectHref}: servers: @@ -891,6 +901,155 @@ paths: summary: Search STAC items with full-featured filtering. tags: - STAC + /collections/{collectionId}/assets: + get: + description: >- + Fetch collection assets of the collection with id `collectionId`. + + These assets do not belong to any item. + operationId: getCollectionAssets + parameters: + - $ref: "#/components/parameters/collectionId" + responses: + "200": + $ref: "#/components/responses/CollectionAssets" + "400": + $ref: "#/components/responses/InvalidParameter" + "404": + $ref: "#/components/responses/NotFound" + "500": + $ref: "#/components/responses/ServerError" + summary: Fetch all collection assets for a collection + tags: + - Data + /collections/{collectionId}/assets/{assetId}: + get: + description: >- + Fetch the collection asset with id `assetId` of the collection with id `collectionId`. + operationId: getCollectionAsset + parameters: + - $ref: "#/components/parameters/collectionId" + - $ref: "#/components/parameters/assetId" + - $ref: "#/components/parameters/IfMatch" + - $ref: "#/components/parameters/IfNoneMatch" + responses: + "200": + $ref: "#/components/responses/CollectionAsset" + "304": + $ref: "#/components/responses/NotModified" + "404": + $ref: "#/components/responses/NotFound" + "412": + $ref: "#/components/responses/PreconditionFailed" + "500": + $ref: "#/components/responses/ServerError" + summary: Fetch a single collection asset + tags: + - Data + put: + summary: Update or create a collection asset + description: >- + Update or create a collection asset with Id `assetId` with a complete asset definition. If the asset doesn't exist it is then created. + + + *Note: to upload an asset file see [Collection Asset Upload Management](#tag/Collection-Asset-Upload-Management)* + operationId: putCollectionAsset + tags: + - Data Management + parameters: + - $ref: "#/components/parameters/collectionId" + - $ref: "#/components/parameters/assetId" + - $ref: "#/components/parameters/IfMatchWrite" + requestBody: + content: + application/json: + schema: + $ref: "#/components/schemas/assetWrite" + responses: + "200": + description: Asset has been successfully updated. + content: + application/json: + schema: + $ref: "#/components/schemas/assetWrite" + "201": + description: Asset has been newly created. + headers: + Location: + description: A link to the asset + schema: + type: string + format: url + content: + application/json: + schema: + $ref: "#/components/schemas/assetWrite" + "400": + $ref: "#/components/responses/BadRequest" + "404": + $ref: "#/components/responses/NotFound" + "412": + $ref: "#/components/responses/PreconditionFailed" + "500": + $ref: "#/components/responses/ServerError" + patch: + summary: Update an existing collection asset by Id with a partial asset definition + description: >- + Use this method to update an existing collection asset. Requires a JSON fragment (containing the fields to be updated) to be submitted. + + + *Note: to upload an asset file see [Collection Asset Upload Management](#tag/Collection-Asset-Upload-Management)* + operationId: patchCollectionAsset + tags: + - Data Management + parameters: + - $ref: "#/components/parameters/collectionId" + - $ref: "#/components/parameters/assetId" + - $ref: "#/components/parameters/IfMatchWrite" + requestBody: + content: + application/json: + schema: + $ref: "#/components/schemas/assetWrite" + responses: + "200": + description: Returns the updated Asset. + content: + application/json: + schema: + $ref: "#/components/schemas/assetWrite" + "400": + $ref: "#/components/responses/BadRequest" + "404": + $ref: "#/components/responses/NotFound" + "412": + $ref: "#/components/responses/PreconditionFailed" + "500": + $ref: "#/components/responses/ServerError" + delete: + summary: Delete an existing collection asset by Id + description: >- + Use this method to delete an existing collection asset. + + **NOTE: Asset file on S3 will be also removed !** + operationId: deleteCollectionAsset + tags: + - Data Management + parameters: + - $ref: "#/components/parameters/collectionId" + - $ref: "#/components/parameters/assetId" + - $ref: "#/components/parameters/IfMatchWrite" + responses: + "200": + $ref: "#/components/responses/DeletedResource" + "400": + $ref: "#/components/responses/BadRequest" + "404": + $ref: "#/components/responses/NotFound" + "412": + $ref: "#/components/responses/PreconditionFailed" + "500": + $ref: "#/components/responses/ServerError" /collections/{collectionId}/items/{featureId}/assets: get: description: >- @@ -941,7 +1100,7 @@ paths: put: summary: Update or create an asset description: >- - Update or create an asset with Id `assetId` with a complete asset definition. If the asset doesn't exists it is then created. + Update or create an asset with Id `assetId` with a complete asset definition. If the asset doesn't exist it is then created. *Note: to upload an asset file see [Asset Upload Management](#tag/Asset-Upload-Management)* @@ -983,12 +1142,12 @@ paths: $ref: "#/components/responses/NotFound" "412": $ref: "#/components/responses/PreconditionFailed" - 5XX: + "500": $ref: "#/components/responses/ServerError" patch: summary: Update an existing asset by Id with a partial asset definition description: >- - Use this method to update an existing asset. Requires a JSON fragment (containing the fields to be updated) be submitted. + Use this method to update an existing asset. Requires a JSON fragment (containing the fields to be updated) to be submitted. *Note: to upload an asset file see [Asset Upload Management](#tag/Asset-Upload-Management)* @@ -1018,7 +1177,7 @@ paths: $ref: "#/components/responses/NotFound" "412": $ref: "#/components/responses/PreconditionFailed" - 5XX: + "500": $ref: "#/components/responses/ServerError" delete: summary: Delete an existing asset by Id @@ -1044,7 +1203,7 @@ paths: $ref: "#/components/responses/NotFound" "412": $ref: "#/components/responses/PreconditionFailed" - 5XX: + "500": $ref: "#/components/responses/ServerError" /collections/{collectionId}/items/{featureId}/assets/{assetId}/uploads: parameters: @@ -1056,7 +1215,7 @@ paths: - Asset Upload Management summary: List all Asset's multipart uploads description: >- - Return a list of all Asset's multipart uploads that are in progress and have been completed or aborted. + Return a list of multipart uploads of assets that are in progress, completed or cancelled. operationId: getAssetUploads parameters: - name: status @@ -1102,7 +1261,7 @@ paths: $ref: "#/components/responses/BadRequest" "404": $ref: "#/components/responses/NotFound" - 5XX: + "500": $ref: "#/components/responses/ServerError" post: tags: @@ -1144,7 +1303,7 @@ paths: application/json: schema: $ref: "#/components/schemas/uploadInProgress" - 5XX: + "500": $ref: "#/components/responses/ServerError" /collections/{collectionId}/items/{featureId}/assets/{assetId}/uploads/{uploadId}: parameters: @@ -1195,6 +1354,7 @@ paths: put: tags: - Asset Upload Management + - Collection Asset Upload Management summary: Upload asset file part description: >- Upload an Asset file part using the presigned url(s) returned by [Create a new Asset's multipart upload](#operation/createAssetUpload). @@ -1323,7 +1483,7 @@ paths: summary: Complete multipart upload operationId: completeMultipartUpload description: >- - Complete the multipart upload process. After completion, the Asset metadata are updated with the new `file:checksum` from the upload and the parts are automatically deleted. The Asset `href` field is also set if it was the first upload. + Complete the multipart upload process. After completion, the Asset metadata are updated with the new `file:checksum` from the upload and the parts are automatically deleted. The assets `href` field is also set if it was the first upload. requestBody: content: application/json: @@ -1349,7 +1509,7 @@ paths: description: No upload in progress "404": $ref: "#/components/responses/NotFound" - 5XX: + "500": $ref: "#/components/responses/ServerError" /collections/{collectionId}/items/{featureId}/assets/{assetId}/uploads/{uploadId}/abort: parameters: @@ -1375,7 +1535,7 @@ paths: $ref: "#/components/responses/BadRequest" "404": $ref: "#/components/responses/NotFound" - 5XX: + "500": $ref: "#/components/responses/ServerError" /collections/{collectionId}/items/{featureId}/assets/{assetId}/uploads/{uploadId}/parts: parameters: @@ -1408,7 +1568,242 @@ paths: $ref: "#/components/responses/BadRequest" "404": $ref: "#/components/responses/NotFound" - 5XX: + "500": + $ref: "#/components/responses/ServerError" + /collections/{collectionId}/assets/{assetId}/uploads: + parameters: + - $ref: "#/components/parameters/collectionId" + - $ref: "#/components/parameters/assetId" + get: + tags: + - Collection Asset Upload Management + summary: List all collection assets multipart uploads + description: >- + Return a list of multipart uploads of collection assets that are in progress, completed or cancelled. + operationId: getCollectionAssetUploads + parameters: + - name: status + in: query + description: Filter the list by status. + schema: + $ref: "#/components/schemas/status" + - $ref: "#/components/parameters/limit" + responses: + "200": + description: List of collection asset uploads + content: + application/json: + schema: + $ref: "#/components/schemas/assetUploads" + example: + uploads: + - upload_id: KrFTuglD.N8ireqry_w3.oQqNwrYI7SfSXpVRiusKah0YigDnusebaJNIUZg4R_No0MMW9FLU2UG5anTW0boTUYVxKfBZWCFXqnQTpjnQEo1K7la39MYpjSTvIbZgnG + status: in-progress + number_parts: 1 + urls: + - url: https://data.geo.admin.ch/ch.swisstopo.pixelkarte-farbe-pk50.noscale/smr200-200-4-2019/smr50-263-2016-2056-kgrs-2.5.tiff + part: 1 + expires: "2019-08-24T14:15:22Z" + created: "2019-08-24T14:15:22Z" + file:checksum: 12200ADEC47F803A8CF1055ED36750B3BA573C79A3AF7DA6D6F5A2AED03EA16AF3BC + - upload_id: KrFTuglD.N8ireqry_w3.oQqNwrYI7SfSXpVRiusKah0YaaegJNIUZg4R_No0MMW9FLU2UG5anTW0boTUYVxKfBZWCFXqnQTpjnQEo1K7la39MYpjSTvIbZgnG + status: completed + number_parts: 1 + created: "2019-08-24T14:15:22Z" + completed: "2019-08-24T14:15:22Z" + file:checksum: 12200ADEC47F803A8CF1055ED36750B3BA573C79A3AF7DA6D6F5A2AED03EA16AF3BC + - upload_id: KrFTuglD.N8ireqry_w3.oQqNwrYI7SfSXpVRiusKah0YigDnuM06hfJNIUZg4R_No0MMW9FLU2UG5anTW0boTUYVxKfBZWCFXqnQTpjnQEo1K7la39MYpjSTvIbZgnG + status: aborted + number_parts: 1 + created: "2019-08-24T14:15:22Z" + aborted: "2019-08-24T14:15:22Z" + file:checksum: 12200ADEC47F803A8CF1055ED36750B3BA573C79A3AF7DA6D6F5A2AED03EA16AF3BC + links: + - rel: next + href: https://data.geo.admin.ch/api/stac/v0.9/collections/ch.swisstopo.pixelkarte-farbe-pk50.noscale/items/smr200-200-4-2019/assets/smr50-263-2016-2056-kgrs-2.5.tiff/uploads?cursor=0d34 + "400": + $ref: "#/components/responses/BadRequest" + "404": + $ref: "#/components/responses/NotFound" + "500": + $ref: "#/components/responses/ServerError" + post: + tags: + - Collection Asset Upload Management + summary: Create a new collection asset multipart upload + description: | + Create collection asset multipart upload. + + A file part must be at least 5 MB except for the last one and at most 5 GB, otherwise the complete operation will fail. + + *Note: in order to provide integrity check during the upload, the base64-encoded 128-bit MD5 digest of each part must be + computed and passed to the create endpoint. Then this digest must also be passed as `Content-MD5` header during the upload.* + operationId: createCollectionAssetUpload + requestBody: + content: + application/json: + schema: + $ref: "#/components/schemas/assetUploadCreate" + responses: + "201": + description: Created collection asset multipart upload + headers: + Location: + description: A link to the collection asset multipart upload object + schema: + type: string + format: url + content: + application/json: + schema: + $ref: "#/components/schemas/assetUploadCreate" + "400": + $ref: "#/components/responses/BadRequest" + "404": + $ref: "#/components/responses/NotFound" + "409": + description: Another upload is already in progress, you first need to complete or abort it before creating a new upload. + content: + application/json: + schema: + $ref: "#/components/schemas/uploadInProgress" + "500": + $ref: "#/components/responses/ServerError" + /collections/{collectionId}/assets/{assetId}/uploads/{uploadId}: + parameters: + - $ref: "#/components/parameters/collectionId" + - $ref: "#/components/parameters/assetId" + - $ref: "#/components/parameters/uploadId" + get: + tags: + - Collection Asset Upload Management + summary: Get a collection assets multipart upload + description: | + Return a collection assets multipart upload. + operationId: getCollectionAssetUpload + parameters: + - $ref: "#/components/parameters/IfMatch" + - $ref: "#/components/parameters/IfNoneMatch" + responses: + "200": + description: Specified collection asset multipart upload. + headers: + ETag: + $ref: "#/components/headers/ETag" + content: + application/json: + schema: + $ref: "#/components/schemas/assetUpload" + examples: + inprogress: + $ref: "#/components/examples/inprogress" + completed: + $ref: "#/components/examples/completed" + aborted: + $ref: "#/components/examples/aborted" + "304": + $ref: "#/components/responses/NotModified" + "404": + $ref: "#/components/responses/NotFound" + "412": + $ref: "#/components/responses/PreconditionFailed" + "500": + $ref: "#/components/responses/ServerError" + /collections/{collectionId}/assets/{assetId}/uploads/{uploadId}/complete: + parameters: + - $ref: "#/components/parameters/collectionId" + - $ref: "#/components/parameters/assetId" + - $ref: "#/components/parameters/uploadId" + post: + tags: + - Collection Asset Upload Management + summary: Complete multipart upload + operationId: completeCollectionAssetMultipartUpload + description: >- + Complete the multipart upload process. After completion, the collection asset metadata are updated with the new `file:checksum` from the upload and the parts are automatically deleted. The assets `href` field is also set if it was the first upload. + requestBody: + content: + application/json: + schema: + $ref: "#/components/schemas/assetCompleteUpload" + responses: + "200": + description: Asset multipart upload completed successfully. + content: + application/json: + schema: + $ref: "#/components/schemas/assetUploadCompleted" + "400": + $ref: "#/components/responses/BadRequest" + "409": + description: The complete operation was already performed. + content: + application/json: + schema: + $ref: "#/components/schemas/exception" + example: + code: 409 + description: No upload in progress + "404": + $ref: "#/components/responses/NotFound" + "500": + $ref: "#/components/responses/ServerError" + /collections/{collectionId}/assets/{assetId}/uploads/{uploadId}/abort: + parameters: + - $ref: "#/components/parameters/collectionId" + - $ref: "#/components/parameters/assetId" + - $ref: "#/components/parameters/uploadId" + post: + tags: + - Collection Asset Upload Management + summary: Abort multipart upload + operationId: abortCollectionAssetMultipartUpload + description: >- + Abort the multipart upload process. All already uploaded parts are automatically deleted. + responses: + "200": + description: Collection asset multipart upload aborted successfully. + content: + application/json: + schema: + $ref: "#/components/schemas/assetUploadAborted" + "400": + $ref: "#/components/responses/BadRequest" + "404": + $ref: "#/components/responses/NotFound" + "500": + $ref: "#/components/responses/ServerError" + /collections/{collectionId}/assets/{assetId}/uploads/{uploadId}/parts: + parameters: + - $ref: "#/components/parameters/collectionId" + - $ref: "#/components/parameters/assetId" + - $ref: "#/components/parameters/uploadId" + get: + tags: + - Collection Asset Upload Management + summary: Get upload parts + operationId: getCollectionAssetUploadParts + description: >- + Return the list of already uploaded parts. + + + ### Pagination + + By default all parts are returned (maximum number of parts being 100). The user can use pagination to reduce the number of returned parts. Pagination is done via the `limit` query parameter (see below). + parameters: + - $ref: "#/components/parameters/limit" + responses: + "200": + description: List of parts already uploaded. + content: + application/json: + schema: + $ref: "#/components/schemas/assetUploadParts" + "400": + $ref: "#/components/responses/BadRequest" + "404": + $ref: "#/components/responses/NotFound" + "500": $ref: "#/components/responses/ServerError" /get-token: servers: @@ -1678,6 +2073,13 @@ components: application/json: schema: $ref: "#/components/schemas/assets" + CollectionAssets: + description: >- + The response is a document consisting of all assets of the collection. + content: + application/json: + schema: + $ref: "#/components/schemas/assets" Asset: description: >- The response is a document consisting of one asset of the feature. @@ -1688,6 +2090,16 @@ components: application/json: schema: $ref: "#/components/schemas/assetWrite" + CollectionAsset: + description: >- + The response is a document consisting of one asset of the collection. + headers: + ETag: + $ref: "#/components/headers/ETag" + content: + application/json: + schema: + $ref: "#/components/schemas/assetWrite" DeletedResource: description: Status of the delete resource content: diff --git a/spec/transaction/components/responses.yaml b/spec/transaction/components/responses.yaml index 61cd3eac..d6c24c2b 100644 --- a/spec/transaction/components/responses.yaml +++ b/spec/transaction/components/responses.yaml @@ -8,6 +8,13 @@ components: application/json: schema: $ref: "./schemas.yaml#/components/schemas/assets" + CollectionAssets: + description: >- + The response is a document consisting of all assets of the collection. + content: + application/json: + schema: + $ref: "./schemas.yaml#/components/schemas/assets" Asset: description: >- The response is a document consisting of one asset of the feature. @@ -18,6 +25,16 @@ components: application/json: schema: $ref: "./schemas.yaml#/components/schemas/assetWrite" + CollectionAsset: + description: >- + The response is a document consisting of one asset of the collection. + headers: + ETag: + $ref: "../../components/headers.yaml#/components/headers/ETag" + content: + application/json: + schema: + $ref: "./schemas.yaml#/components/schemas/assetWrite" DeletedResource: description: Status of the delete resource content: @@ -44,4 +61,4 @@ components: rel: parent required: - code - - links \ No newline at end of file + - links diff --git a/spec/transaction/paths.yaml b/spec/transaction/paths.yaml index 36178240..a59436dc 100644 --- a/spec/transaction/paths.yaml +++ b/spec/transaction/paths.yaml @@ -7,7 +7,7 @@ paths: summary: Update or create a collection description: >- Update or create a collection with Id `collectionId` with a complete collection definition. - If the collection doesn't exists it is then created. + If the collection doesn't exist it is created. operationId: updateCollection parameters: - $ref: "../components/parameters.yaml#/components/parameters/collectionId" @@ -107,13 +107,165 @@ paths: $ref: "../components/responses.yaml#/components/responses/PreconditionFailed" "500": $ref: "../components/responses.yaml#/components/responses/ServerError" + "/collections/{collectionId}/assets": + get: + description: >- + Fetch collection assets of the collection with id `collectionId`. + + These assets do not belong to any item. + operationId: getCollectionAssets + parameters: + - $ref: "../components/parameters.yaml#/components/parameters/collectionId" + responses: + "200": + $ref: "./components/responses.yaml#/components/responses/CollectionAssets" + "400": + $ref: "../components/responses.yaml#/components/responses/InvalidParameter" + "404": + $ref: "../components/responses.yaml#/components/responses/NotFound" + "500": + $ref: "../components/responses.yaml#/components/responses/ServerError" + summary: Fetch all collection assets for a collection + tags: + - Data + + "/collections/{collectionId}/assets/{assetId}": + get: + description: >- + Fetch the collection asset with id `assetId` of the collection with id `collectionId`. + operationId: getCollectionAsset + parameters: + - $ref: "../components/parameters.yaml#/components/parameters/collectionId" + - $ref: "./components/parameters.yaml#/components/parameters/assetId" + - $ref: "../components/parameters.yaml#/components/parameters/IfMatch" + - $ref: "../components/parameters.yaml#/components/parameters/IfNoneMatch" + responses: + "200": + $ref: "./components/responses.yaml#/components/responses/CollectionAsset" + "304": + $ref: "../components/responses.yaml#/components/responses/NotModified" + "404": + $ref: "../components/responses.yaml#/components/responses/NotFound" + "412": + $ref: "../components/responses.yaml#/components/responses/PreconditionFailed" + "500": + $ref: "../components/responses.yaml#/components/responses/ServerError" + summary: Fetch a single collection asset + tags: + - Data + put: + summary: Update or create a collection asset + description: >- + Update or create a collection asset with Id `assetId` with a complete asset definition. + If the asset doesn't exist it is then created. + + + *Note: to upload an asset file see [Collection Asset Upload Management](#tag/Collection-Asset-Upload-Management)* + operationId: putCollectionAsset + tags: + - Data Management + parameters: + - $ref: "../components/parameters.yaml#/components/parameters/collectionId" + - $ref: "./components/parameters.yaml#/components/parameters/assetId" + - $ref: "./components/parameters.yaml#/components/parameters/IfMatchWrite" + requestBody: + content: + application/json: + schema: + $ref: "./components/schemas.yaml#/components/schemas/assetWrite" + responses: + "200": + description: Asset has been successfully updated. + content: + application/json: + schema: + $ref: "./components/schemas.yaml#/components/schemas/assetWrite" + "201": + description: Asset has been newly created. + headers: + Location: + description: A link to the asset + schema: + type: string + format: url + content: + application/json: + schema: + $ref: "./components/schemas.yaml#/components/schemas/assetWrite" + "400": + $ref: "../components/responses.yaml#/components/responses/BadRequest" + "404": + $ref: "../components/responses.yaml#/components/responses/NotFound" + "412": + $ref: "../components/responses.yaml#/components/responses/PreconditionFailed" + "500": + $ref: "../components/responses.yaml#/components/responses/ServerError" + patch: + summary: Update an existing collection asset by Id with a partial asset definition + description: >- + Use this method to update an existing collection asset. Requires a JSON + fragment (containing the fields to be updated) to be submitted. + + + *Note: to upload an asset file see [Collection Asset Upload Management](#tag/Collection-Asset-Upload-Management)* + operationId: patchCollectionAsset + tags: + - Data Management + parameters: + - $ref: "../components/parameters.yaml#/components/parameters/collectionId" + - $ref: "./components/parameters.yaml#/components/parameters/assetId" + - $ref: "./components/parameters.yaml#/components/parameters/IfMatchWrite" + requestBody: + content: + application/json: + schema: + $ref: "./components/schemas.yaml#/components/schemas/assetWrite" + responses: + "200": + description: Returns the updated Asset. + content: + application/json: + schema: + $ref: "./components/schemas.yaml#/components/schemas/assetWrite" + "400": + $ref: "../components/responses.yaml#/components/responses/BadRequest" + "404": + $ref: "../components/responses.yaml#/components/responses/NotFound" + "412": + $ref: "../components/responses.yaml#/components/responses/PreconditionFailed" + "500": + $ref: "../components/responses.yaml#/components/responses/ServerError" + delete: + summary: Delete an existing collection asset by Id + description: >- + Use this method to delete an existing collection asset. + + **NOTE: Asset file on S3 will be also removed !** + operationId: deleteCollectionAsset + tags: + - Data Management + parameters: + - $ref: "../components/parameters.yaml#/components/parameters/collectionId" + - $ref: "./components/parameters.yaml#/components/parameters/assetId" + - $ref: "./components/parameters.yaml#/components/parameters/IfMatchWrite" + responses: + "200": + $ref: "./components/responses.yaml#/components/responses/DeletedResource" + "400": + $ref: "../components/responses.yaml#/components/responses/BadRequest" + "404": + $ref: "../components/responses.yaml#/components/responses/NotFound" + "412": + $ref: "../components/responses.yaml#/components/responses/PreconditionFailed" + "500": + $ref: "../components/responses.yaml#/components/responses/ServerError" "/collections/{collectionId}/items/{featureId}": put: summary: Update or create a feature description: >- Update or create a feature with Id `featureId` with a complete feature - definition. If the feature doesn't exists it is then created. + definition. If the feature doesn't exist it is then created. *NOTE: Optional fields that are not part of the PUT payload, will be erased @@ -181,13 +333,13 @@ paths: $ref: "../components/responses.yaml#/components/responses/NotFound" "412": $ref: "../components/responses.yaml#/components/responses/PreconditionFailed" - "5XX": + "500": $ref: "../components/responses.yaml#/components/responses/ServerError" patch: summary: Update an existing feature by Id with a partial item definition description: >- Use this method to update an existing feature. Requires a JSON - fragment (containing the fields to be updated) be submitted. + fragment (containing the fields to be updated) to be submitted. operationId: patchFeature tags: - Data Management @@ -227,7 +379,7 @@ paths: $ref: "../components/responses.yaml#/components/responses/NotFound" "412": $ref: "../components/responses.yaml#/components/responses/PreconditionFailed" - "5XX": + "500": $ref: "../components/responses.yaml#/components/responses/ServerError" delete: summary: Delete an existing feature by Id @@ -250,7 +402,7 @@ paths: $ref: "../components/responses.yaml#/components/responses/NotFound" "412": $ref: "../components/responses.yaml#/components/responses/PreconditionFailed" - "5XX": + "500": $ref: "../components/responses.yaml#/components/responses/ServerError" "/collections/{collectionId}/items/{featureId}/assets": get: @@ -305,7 +457,7 @@ paths: summary: Update or create an asset description: >- Update or create an asset with Id `assetId` with a complete asset definition. - If the asset doesn't exists it is then created. + If the asset doesn't exist it is then created. *Note: to upload an asset file see [Asset Upload Management](#tag/Asset-Upload-Management)* @@ -347,13 +499,13 @@ paths: $ref: "../components/responses.yaml#/components/responses/NotFound" "412": $ref: "../components/responses.yaml#/components/responses/PreconditionFailed" - "5XX": + "500": $ref: "../components/responses.yaml#/components/responses/ServerError" patch: summary: Update an existing asset by Id with a partial asset definition description: >- Use this method to update an existing asset. Requires a JSON - fragment (containing the fields to be updated) be submitted. + fragment (containing the fields to be updated) to be submitted. *Note: to upload an asset file see [Asset Upload Management](#tag/Asset-Upload-Management)* @@ -383,7 +535,7 @@ paths: $ref: "../components/responses.yaml#/components/responses/NotFound" "412": $ref: "../components/responses.yaml#/components/responses/PreconditionFailed" - "5XX": + "500": $ref: "../components/responses.yaml#/components/responses/ServerError" delete: summary: Delete an existing asset by Id @@ -409,7 +561,7 @@ paths: $ref: "../components/responses.yaml#/components/responses/NotFound" "412": $ref: "../components/responses.yaml#/components/responses/PreconditionFailed" - "5XX": + "500": $ref: "../components/responses.yaml#/components/responses/ServerError" "/collections/{collectionId}/items/{featureId}/assets/{assetId}/uploads": @@ -422,8 +574,7 @@ paths: - Asset Upload Management summary: List all Asset's multipart uploads description: >- - Return a list of all Asset's multipart uploads that are in progress and have been completed - or aborted. + Return a list of multipart uploads of assets that are in progress, completed or cancelled. operationId: getAssetUploads parameters: - name: status @@ -469,7 +620,7 @@ paths: $ref: "../components/responses.yaml#/components/responses/BadRequest" "404": $ref: "../components/responses.yaml#/components/responses/NotFound" - "5XX": + "500": $ref: "../components/responses.yaml#/components/responses/ServerError" post: tags: @@ -511,7 +662,7 @@ paths: application/json: schema: $ref: "./schemas.yaml#/components/schemas/uploadInProgress" - "5XX": + "500": $ref: "../components/responses.yaml#/components/responses/ServerError" "/collections/{collectionId}/items/{featureId}/assets/{assetId}/uploads/{uploadId}": parameters: @@ -562,6 +713,7 @@ paths: put: tags: - Asset Upload Management + - Collection Asset Upload Management summary: Upload asset file part description: >- Upload an Asset file part using the presigned url(s) returned by @@ -699,7 +851,7 @@ paths: description: >- Complete the multipart upload process. After completion, the Asset metadata are updated with the new `file:checksum` from the upload and the parts are automatically deleted. - The Asset `href` field is also set if it was the first upload. + The assets `href` field is also set if it was the first upload. requestBody: content: application/json: @@ -725,7 +877,7 @@ paths: description: No upload in progress "404": $ref: "../components/responses.yaml#/components/responses/NotFound" - "5XX": + "500": $ref: "../components/responses.yaml#/components/responses/ServerError" "/collections/{collectionId}/items/{featureId}/assets/{assetId}/uploads/{uploadId}/abort": parameters: @@ -751,7 +903,7 @@ paths: $ref: "../components/responses.yaml#/components/responses/BadRequest" "404": $ref: "../components/responses.yaml#/components/responses/NotFound" - "5XX": + "500": $ref: "../components/responses.yaml#/components/responses/ServerError" "/collections/{collectionId}/items/{featureId}/assets/{assetId}/uploads/{uploadId}/parts": parameters: @@ -786,9 +938,251 @@ paths: $ref: "../components/responses.yaml#/components/responses/BadRequest" "404": $ref: "../components/responses.yaml#/components/responses/NotFound" - "5XX": + "500": $ref: "../components/responses.yaml#/components/responses/ServerError" + "/collections/{collectionId}/assets/{assetId}/uploads": + parameters: + - $ref: "../components/parameters.yaml#/components/parameters/collectionId" + - $ref: "./components/parameters.yaml#/components/parameters/assetId" + get: + tags: + - Collection Asset Upload Management + summary: List all collection assets multipart uploads + description: >- + Return a list of multipart uploads of collection assets that are in progress, completed or cancelled. + operationId: getCollectionAssetUploads + parameters: + - name: status + in: query + description: Filter the list by status. + schema: + $ref: "./components/schemas.yaml#/components/schemas/status" + - $ref: "../components/parameters.yaml#/components/parameters/limit" + responses: + 200: + description: List of collection asset uploads + content: + application/json: + schema: + $ref: "./components/schemas.yaml#/components/schemas/assetUploads" + example: + uploads: + - upload_id: KrFTuglD.N8ireqry_w3.oQqNwrYI7SfSXpVRiusKah0YigDnusebaJNIUZg4R_No0MMW9FLU2UG5anTW0boTUYVxKfBZWCFXqnQTpjnQEo1K7la39MYpjSTvIbZgnG + status: in-progress + number_parts: 1 + urls: + - url: https://data.geo.admin.ch/ch.swisstopo.pixelkarte-farbe-pk50.noscale/smr200-200-4-2019/smr50-263-2016-2056-kgrs-2.5.tiff + part: 1 + expires: "2019-08-24T14:15:22Z" + created: "2019-08-24T14:15:22Z" + file:checksum: 12200ADEC47F803A8CF1055ED36750B3BA573C79A3AF7DA6D6F5A2AED03EA16AF3BC + - upload_id: KrFTuglD.N8ireqry_w3.oQqNwrYI7SfSXpVRiusKah0YaaegJNIUZg4R_No0MMW9FLU2UG5anTW0boTUYVxKfBZWCFXqnQTpjnQEo1K7la39MYpjSTvIbZgnG + status: completed + number_parts: 1 + created: "2019-08-24T14:15:22Z" + completed: "2019-08-24T14:15:22Z" + file:checksum: 12200ADEC47F803A8CF1055ED36750B3BA573C79A3AF7DA6D6F5A2AED03EA16AF3BC + - upload_id: KrFTuglD.N8ireqry_w3.oQqNwrYI7SfSXpVRiusKah0YigDnuM06hfJNIUZg4R_No0MMW9FLU2UG5anTW0boTUYVxKfBZWCFXqnQTpjnQEo1K7la39MYpjSTvIbZgnG + status: aborted + number_parts: 1 + created: "2019-08-24T14:15:22Z" + aborted: "2019-08-24T14:15:22Z" + file:checksum: 12200ADEC47F803A8CF1055ED36750B3BA573C79A3AF7DA6D6F5A2AED03EA16AF3BC + links: + - rel: next + href: https://data.geo.admin.ch/api/stac/v0.9/collections/ch.swisstopo.pixelkarte-farbe-pk50.noscale/items/smr200-200-4-2019/assets/smr50-263-2016-2056-kgrs-2.5.tiff/uploads?cursor=0d34 + "400": + $ref: "../components/responses.yaml#/components/responses/BadRequest" + "404": + $ref: "../components/responses.yaml#/components/responses/NotFound" + "500": + $ref: "../components/responses.yaml#/components/responses/ServerError" + post: + tags: + - Collection Asset Upload Management + summary: Create a new collection asset multipart upload + description: | + Create collection asset multipart upload. + + A file part must be at least 5 MB except for the last one and at most 5 GB, otherwise the complete operation will fail. + + *Note: in order to provide integrity check during the upload, the base64-encoded 128-bit MD5 digest of each part must be + computed and passed to the create endpoint. Then this digest must also be passed as `Content-MD5` header during the upload.* + operationId: createCollectionAssetUpload + requestBody: + content: + application/json: + schema: + $ref: "./components/schemas.yaml#/components/schemas/assetUploadCreate" + responses: + 201: + description: Created collection asset multipart upload + headers: + Location: + description: A link to the collection asset multipart upload object + schema: + type: string + format: url + content: + application/json: + schema: + $ref: "./components/schemas.yaml#/components/schemas/assetUploadCreate" + "400": + $ref: "../components/responses.yaml#/components/responses/BadRequest" + "404": + $ref: "../components/responses.yaml#/components/responses/NotFound" + "409": + description: Another upload is already in progress, you first need to complete or abort it before creating a new upload. + content: + application/json: + schema: + $ref: "./schemas.yaml#/components/schemas/uploadInProgress" + "500": + $ref: "../components/responses.yaml#/components/responses/ServerError" + "/collections/{collectionId}/assets/{assetId}/uploads/{uploadId}": + parameters: + - $ref: "../components/parameters.yaml#/components/parameters/collectionId" + - $ref: "./components/parameters.yaml#/components/parameters/assetId" + - $ref: "./components/parameters.yaml#/components/parameters/uploadId" + get: + tags: + - Collection Asset Upload Management + summary: Get a collection assets multipart upload + description: | + Return a collection assets multipart upload. + operationId: getCollectionAssetUpload + parameters: + - $ref: "../components/parameters.yaml#/components/parameters/IfMatch" + - $ref: "../components/parameters.yaml#/components/parameters/IfNoneMatch" + responses: + "200": + description: Specified collection asset multipart upload. + headers: + ETag: + $ref: "../components/headers.yaml#/components/headers/ETag" + content: + application/json: + schema: + $ref: "./components/schemas.yaml#/components/schemas/assetUpload" + examples: + inprogress: + $ref: "./components/examples.yaml#/components/examples/inprogress" + completed: + $ref: "./components/examples.yaml#/components/examples/completed" + aborted: + $ref: "./components/examples.yaml#/components/examples/aborted" + "304": + $ref: "../components/responses.yaml#/components/responses/NotModified" + "404": + $ref: "../components/responses.yaml#/components/responses/NotFound" + "412": + $ref: "../components/responses.yaml#/components/responses/PreconditionFailed" + "500": + $ref: "../components/responses.yaml#/components/responses/ServerError" + + "/collections/{collectionId}/assets/{assetId}/uploads/{uploadId}/complete": + parameters: + - $ref: "../components/parameters.yaml#/components/parameters/collectionId" + - $ref: "./components/parameters.yaml#/components/parameters/assetId" + - $ref: "./components/parameters.yaml#/components/parameters/uploadId" + post: + tags: + - Collection Asset Upload Management + summary: Complete multipart upload + operationId: completeCollectionAssetMultipartUpload + description: >- + Complete the multipart upload process. After completion, the collection asset metadata are updated + with the new `file:checksum` from the upload and the parts are automatically deleted. + The assets `href` field is also set if it was the first upload. + requestBody: + content: + application/json: + schema: + $ref: "./components/schemas.yaml#/components/schemas/assetCompleteUpload" + responses: + "200": + description: Asset multipart upload completed successfully. + content: + application/json: + schema: + $ref: "./components/schemas.yaml#/components/schemas/assetUploadCompleted" + "400": + $ref: "../components/responses.yaml#/components/responses/BadRequest" + "409": + description: The complete operation was already performed. + content: + application/json: + schema: + $ref: "./schemas.yaml#/components/schemas/exception" + example: + code: 409 + description: No upload in progress + "404": + $ref: "../components/responses.yaml#/components/responses/NotFound" + "500": + $ref: "../components/responses.yaml#/components/responses/ServerError" + "/collections/{collectionId}/assets/{assetId}/uploads/{uploadId}/abort": + parameters: + - $ref: "../components/parameters.yaml#/components/parameters/collectionId" + - $ref: "./components/parameters.yaml#/components/parameters/assetId" + - $ref: "./components/parameters.yaml#/components/parameters/uploadId" + post: + tags: + - Collection Asset Upload Management + summary: Abort multipart upload + operationId: abortCollectionAssetMultipartUpload + description: >- + Abort the multipart upload process. All already uploaded parts are automatically deleted. + responses: + "200": + description: Collection asset multipart upload aborted successfully. + content: + application/json: + schema: + $ref: "./components/schemas.yaml#/components/schemas/assetUploadAborted" + "400": + $ref: "../components/responses.yaml#/components/responses/BadRequest" + "404": + $ref: "../components/responses.yaml#/components/responses/NotFound" + "500": + $ref: "../components/responses.yaml#/components/responses/ServerError" + "/collections/{collectionId}/assets/{assetId}/uploads/{uploadId}/parts": + parameters: + - $ref: "../components/parameters.yaml#/components/parameters/collectionId" + - $ref: "./components/parameters.yaml#/components/parameters/assetId" + - $ref: "./components/parameters.yaml#/components/parameters/uploadId" + get: + tags: + - Collection Asset Upload Management + summary: Get upload parts + operationId: getCollectionAssetUploadParts + description: >- + Return the list of already uploaded parts. + + + ### Pagination + + By default all parts are returned (maximum number of parts being 100). The user can + use pagination to reduce the number of returned parts. Pagination is done via the `limit` + query parameter (see below). + parameters: + - $ref: "../components/parameters.yaml#/components/parameters/limit" + responses: + "200": + description: List of parts already uploaded. + content: + application/json: + schema: + $ref: "./components/schemas.yaml#/components/schemas/assetUploadParts" + "400": + $ref: "../components/responses.yaml#/components/responses/BadRequest" + "404": + $ref: "../components/responses.yaml#/components/responses/NotFound" + "500": + $ref: "../components/responses.yaml#/components/responses/ServerError" + + "/get-token": servers: - url: http://data.geo.admin.ch/api/stac/ diff --git a/spec/transaction/tags.yaml b/spec/transaction/tags.yaml index 8e7b2516..ab05f77d 100644 --- a/spec/transaction/tags.yaml +++ b/spec/transaction/tags.yaml @@ -99,7 +99,7 @@ tags: (1) Use the [create new upload](#tag/Asset-Upload-Management/operation/createAssetUpload) request to start a new upload. It will return a list of urls. - (2) Use the urls to [upload asset file part](#tag/Asset-Upload-Management/operation/uploadAssetFilePart). Do this for each file part. + (2) Use the urls to [upload asset file part](#tag/Asset-Upload-Management/operation/uploadAssetFilePart). Do this for each file part. You may also upload multiple parts in parallel. (3) Once all parts have been uploaded, execute the [complete the upload](#tag/Asset-Upload-Management/operation/completeMultipartUpload) request. The new asset file be available to users after you have successfully completed the upload. @@ -283,6 +283,17 @@ tags: ``` See https://aws.amazon.com/premiumsupport/knowledge-center/data-integrity-s3/ for other examples on how to compute the base64 MD5 of a part. + - name: Collection Asset Upload Management + description: | + Collection Asset files are uploaded via the STAC API using the API requests described in this chapter. + + The flow of the requests is the same as for assets that belong to features, which is described under [Asset Upload Management](#tag/Asset-Upload-Management) + - [Basic steps to upload assets](#section/Basic-Steps-to-upload-assets) + - [Detailed example of asset upload](#section/Detailed-example-of-asset-upload) + - [Authentication](#section/Authentication) + - [Compression](#section/Compression) + - [Example](#section/Example) + - name: Authentication description: | All write requests require authentication. There is currently three type of supported authentications: From cc331f241b59fbc798a65bb95b2fb6a5ef739fd5 Mon Sep 17 00:00:00 2001 From: Benjamin Sugden Date: Thu, 5 Sep 2024 17:15:44 +0200 Subject: [PATCH 17/17] PB-755: Fix typos in api spec --- spec/transaction/paths.yaml | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/spec/transaction/paths.yaml b/spec/transaction/paths.yaml index a59436dc..da128b78 100644 --- a/spec/transaction/paths.yaml +++ b/spec/transaction/paths.yaml @@ -631,8 +631,8 @@ paths: A file part must be at least 5 MB except for the last one and at most 5 GB, otherwise the complete operation will fail. - *Note: in order to provide integrity check during the upload, the base64-encoded 128-bit MD5 digest of each part must be - computed and passed in the create endpoint. Then this digest must also be passed as `Content-MD5` header during the upload.* + *Note: in order to facilitate an integrity check during the upload, the base64-encoded 128-bit MD5 digest of each part must be + computed and passed to the create endpoint. Then this digest must also be passed as `Content-MD5` header during the upload.* operationId: createAssetUpload requestBody: content: @@ -657,7 +657,7 @@ paths: "404": $ref: "../components/responses.yaml#/components/responses/NotFound" "409": - description: Another upload is already in progress, you need first to complete or abort it before creating a new upload. + description: Another upload is already in progress, you first need to complete or abort it before creating a new upload. content: application/json: schema: @@ -682,7 +682,7 @@ paths: - $ref: "../components/parameters.yaml#/components/parameters/IfNoneMatch" responses: "200": - description: Asset's multipart upload description. + description: The multipart upload object. headers: ETag: $ref: "../components/headers.yaml#/components/headers/ETag" @@ -851,7 +851,7 @@ paths: description: >- Complete the multipart upload process. After completion, the Asset metadata are updated with the new `file:checksum` from the upload and the parts are automatically deleted. - The assets `href` field is also set if it was the first upload. + The asset's `href` field is also set if it was the first upload. requestBody: content: application/json: @@ -1007,7 +1007,7 @@ paths: A file part must be at least 5 MB except for the last one and at most 5 GB, otherwise the complete operation will fail. - *Note: in order to provide integrity check during the upload, the base64-encoded 128-bit MD5 digest of each part must be + *Note: in order to facilitate an integrity check during the upload, the base64-encoded 128-bit MD5 digest of each part must be computed and passed to the create endpoint. Then this digest must also be passed as `Content-MD5` header during the upload.* operationId: createCollectionAssetUpload requestBody: @@ -1057,7 +1057,7 @@ paths: - $ref: "../components/parameters.yaml#/components/parameters/IfNoneMatch" responses: "200": - description: Specified collection asset multipart upload. + description: The multipart upload object. headers: ETag: $ref: "../components/headers.yaml#/components/headers/ETag" @@ -1094,7 +1094,7 @@ paths: description: >- Complete the multipart upload process. After completion, the collection asset metadata are updated with the new `file:checksum` from the upload and the parts are automatically deleted. - The assets `href` field is also set if it was the first upload. + The asset's `href` field is also set if it was the first upload. requestBody: content: application/json: