diff --git a/app/stac_api/management/commands/list_asset_uploads.py b/app/stac_api/management/commands/list_asset_uploads.py index ff4d1b83..079ac3ec 100644 --- a/app/stac_api/management/commands/list_asset_uploads.py +++ b/app/stac_api/management/commands/list_asset_uploads.py @@ -6,7 +6,7 @@ from stac_api.models import AssetUpload from stac_api.s3_multipart_upload import MultipartUpload -from stac_api.serializers.serializers import AssetUploadSerializer +from stac_api.serializers.upload import AssetUploadSerializer from stac_api.utils import CommandHandler from stac_api.utils import get_asset_path diff --git a/app/stac_api/serializers/collection.py b/app/stac_api/serializers/collection.py index 1968e7fb..b01cd722 100644 --- a/app/stac_api/serializers/collection.py +++ b/app/stac_api/serializers/collection.py @@ -8,8 +8,8 @@ from stac_api.models import CollectionAsset from stac_api.models import CollectionLink from stac_api.models import Provider -from stac_api.serializers.serializers import AssetsDictSerializer -from stac_api.serializers.serializers import HrefField +from stac_api.serializers.serializers_utils import AssetsDictSerializer +from stac_api.serializers.serializers_utils import HrefField from stac_api.serializers.serializers_utils import NonNullModelSerializer from stac_api.serializers.serializers_utils import UpsertModelSerializerMixin from stac_api.serializers.serializers_utils import get_relation_links diff --git a/app/stac_api/serializers/item.py b/app/stac_api/serializers/item.py index 7c545855..eb067475 100644 --- a/app/stac_api/serializers/item.py +++ b/app/stac_api/serializers/item.py @@ -9,8 +9,8 @@ from stac_api.models import Asset from stac_api.models import Item from stac_api.models import ItemLink -from stac_api.serializers.serializers import AssetsDictSerializer -from stac_api.serializers.serializers import HrefField +from stac_api.serializers.serializers_utils import AssetsDictSerializer +from stac_api.serializers.serializers_utils import HrefField from stac_api.serializers.serializers_utils import NonNullModelSerializer from stac_api.serializers.serializers_utils import UpsertModelSerializerMixin from stac_api.serializers.serializers_utils import get_relation_links diff --git a/app/stac_api/serializers/serializers.py b/app/stac_api/serializers/serializers.py index ecb87a3e..4d1aba7e 100644 --- a/app/stac_api/serializers/serializers.py +++ b/app/stac_api/serializers/serializers.py @@ -6,24 +6,14 @@ from django.utils.translation import gettext_lazy as _ from rest_framework import serializers -from rest_framework.utils.serializer_helpers import ReturnDict from rest_framework.validators import UniqueValidator -from stac_api.models import AssetUpload -from stac_api.models import CollectionAssetUpload from stac_api.models import LandingPage from stac_api.models import LandingPageLink -from stac_api.serializers.serializers_utils import DictSerializer -from stac_api.serializers.serializers_utils import NonNullModelSerializer -from stac_api.utils import build_asset_href from stac_api.utils import get_browser_url from stac_api.utils import get_stac_version from stac_api.utils import get_url from stac_api.utils import is_api_version_1 -from stac_api.utils import isoformat -from stac_api.validators import validate_checksum_multihash_sha256 -from stac_api.validators import validate_content_encoding -from stac_api.validators import validate_md5_parts from stac_api.validators import validate_name logger = logging.getLogger(__name__) @@ -138,244 +128,3 @@ def to_representation(self, instance): ]), ] return representation - - -class AssetsDictSerializer(DictSerializer): - '''Assets serializer list to dictionary - - This serializer returns an asset dictionary with the asset name as keys. - ''' - # pylint: disable=abstract-method - key_identifier = 'id' - - -class HrefField(serializers.Field): - '''Special Href field for Assets''' - - # pylint: disable=abstract-method - - def to_representation(self, value): - # build an absolute URL from the file path - request = self.context.get("request") - path = value.name - - if value.instance.is_external: - return path - return build_asset_href(request, path) - - def to_internal_value(self, data): - return data - - -class AssetUploadListSerializer(serializers.ListSerializer): - # pylint: disable=abstract-method - - def to_representation(self, data): - return {'uploads': super().to_representation(data)} - - @property - def data(self): - ret = super(serializers.ListSerializer, self).data - return ReturnDict(ret, serializer=self) - - -class UploadPartSerializer(serializers.Serializer): - '''This serializer is used to serialize the data from/to the S3 API. - ''' - # pylint: disable=abstract-method - etag = serializers.CharField(source='ETag', allow_blank=False, required=True) - part_number = serializers.IntegerField( - source='PartNumber', min_value=1, max_value=100, required=True, allow_null=False - ) - modified = serializers.DateTimeField(source='LastModified', required=False, allow_null=True) - size = serializers.IntegerField(source='Size', allow_null=True, required=False) - - -class AssetUploadSerializer(NonNullModelSerializer): - - class Meta: - model = AssetUpload - list_serializer_class = AssetUploadListSerializer - fields = [ - 'upload_id', - 'status', - 'created', - 'checksum_multihash', - 'completed', - 'aborted', - 'number_parts', - 'md5_parts', - 'urls', - 'ended', - 'parts', - 'update_interval', - 'content_encoding' - ] - - checksum_multihash = serializers.CharField( - source='checksum_multihash', - max_length=255, - required=True, - allow_blank=False, - validators=[validate_checksum_multihash_sha256] - ) - md5_parts = serializers.JSONField(required=True) - update_interval = serializers.IntegerField( - required=False, allow_null=False, min_value=-1, max_value=3600, default=-1 - ) - content_encoding = serializers.CharField( - required=False, - allow_null=False, - allow_blank=False, - min_length=1, - max_length=32, - default='', - validators=[validate_content_encoding] - ) - - # write only fields - ended = serializers.DateTimeField(write_only=True, required=False) - parts = serializers.ListField( - child=UploadPartSerializer(), write_only=True, allow_empty=False, required=False - ) - - # Read only fields - upload_id = serializers.CharField(read_only=True) - created = serializers.DateTimeField(read_only=True) - urls = serializers.JSONField(read_only=True) - completed = serializers.SerializerMethodField() - aborted = serializers.SerializerMethodField() - - def validate(self, attrs): - # get partial from kwargs (if partial true and no md5 : ok, if false no md5 : error) - # Check the md5 parts length - if attrs.get('md5_parts') is not None: - validate_md5_parts(attrs['md5_parts'], attrs['number_parts']) - elif not self.partial: - raise serializers.ValidationError( - detail={'md5_parts': _('md5_parts parameter is missing')}, code='missing' - ) - return attrs - - def get_completed(self, obj): - if obj.status == AssetUpload.Status.COMPLETED: - return isoformat(obj.ended) - return None - - def get_aborted(self, obj): - if obj.status == AssetUpload.Status.ABORTED: - return isoformat(obj.ended) - return None - - def get_fields(self): - fields = super().get_fields() - # This is a hack to allow fields with special characters - fields['file:checksum'] = fields.pop('checksum_multihash') - - # Older versions of the api still use different name - request = self.context.get('request') - if not is_api_version_1(request): - fields['checksum:multihash'] = fields.pop('file:checksum') - return fields - - -class AssetUploadPartsSerializer(serializers.Serializer): - '''S3 list_parts response serializer''' - - # pylint: disable=abstract-method - - class Meta: - list_serializer_class = AssetUploadListSerializer - - # Read only fields - parts = serializers.ListField( - source='Parts', child=UploadPartSerializer(), default=list, read_only=True - ) - - -class CollectionAssetUploadSerializer(NonNullModelSerializer): - - class Meta: - model = CollectionAssetUpload - list_serializer_class = AssetUploadListSerializer - fields = [ - 'upload_id', - 'status', - 'created', - 'checksum_multihash', - 'completed', - 'aborted', - 'number_parts', - 'md5_parts', - 'urls', - 'ended', - 'parts', - 'update_interval', - 'content_encoding' - ] - - checksum_multihash = serializers.CharField( - source='checksum_multihash', - max_length=255, - required=True, - allow_blank=False, - validators=[validate_checksum_multihash_sha256] - ) - md5_parts = serializers.JSONField(required=True) - update_interval = serializers.IntegerField( - required=False, allow_null=False, min_value=-1, max_value=3600, default=-1 - ) - content_encoding = serializers.CharField( - required=False, - allow_null=False, - allow_blank=False, - min_length=1, - max_length=32, - default='', - validators=[validate_content_encoding] - ) - - # write only fields - ended = serializers.DateTimeField(write_only=True, required=False) - parts = serializers.ListField( - child=UploadPartSerializer(), write_only=True, allow_empty=False, required=False - ) - - # Read only fields - upload_id = serializers.CharField(read_only=True) - created = serializers.DateTimeField(read_only=True) - urls = serializers.JSONField(read_only=True) - completed = serializers.SerializerMethodField() - aborted = serializers.SerializerMethodField() - - def validate(self, attrs): - # get partial from kwargs (if partial true and no md5 : ok, if false no md5 : error) - # Check the md5 parts length - if attrs.get('md5_parts') is not None: - validate_md5_parts(attrs['md5_parts'], attrs['number_parts']) - elif not self.partial: - raise serializers.ValidationError( - detail={'md5_parts': _('md5_parts parameter is missing')}, code='missing' - ) - return attrs - - def get_completed(self, obj): - if obj.status == CollectionAssetUpload.Status.COMPLETED: - return isoformat(obj.ended) - return None - - def get_aborted(self, obj): - if obj.status == CollectionAssetUpload.Status.ABORTED: - return isoformat(obj.ended) - return None - - def get_fields(self): - fields = super().get_fields() - # This is a hack to allow fields with special characters - fields['file:checksum'] = fields.pop('checksum_multihash') - - # Older versions of the api still use different name - request = self.context.get('request') - if not is_api_version_1(request): - fields['checksum:multihash'] = fields.pop('file:checksum') - return fields diff --git a/app/stac_api/serializers/serializers_utils.py b/app/stac_api/serializers/serializers_utils.py index 68b0a2b6..70ae5220 100644 --- a/app/stac_api/serializers/serializers_utils.py +++ b/app/stac_api/serializers/serializers_utils.py @@ -4,6 +4,7 @@ from rest_framework import serializers from rest_framework.utils.serializer_helpers import ReturnDict +from stac_api.utils import build_asset_href from stac_api.utils import get_browser_url from stac_api.utils import get_url @@ -311,3 +312,30 @@ def to_representation(self, data): def data(self): ret = super(serializers.ListSerializer, self).data return ReturnDict(ret, serializer=self) + + +class AssetsDictSerializer(DictSerializer): + '''Assets serializer list to dictionary + + This serializer returns an asset dictionary with the asset name as keys. + ''' + # pylint: disable=abstract-method + key_identifier = 'id' + + +class HrefField(serializers.Field): + '''Special Href field for Assets''' + + # pylint: disable=abstract-method + + def to_representation(self, value): + # build an absolute URL from the file path + request = self.context.get("request") + path = value.name + + if value.instance.is_external: + return path + return build_asset_href(request, path) + + def to_internal_value(self, data): + return data diff --git a/app/stac_api/serializers/upload.py b/app/stac_api/serializers/upload.py new file mode 100644 index 00000000..286d4a90 --- /dev/null +++ b/app/stac_api/serializers/upload.py @@ -0,0 +1,231 @@ +import logging + +from django.utils.translation import gettext_lazy as _ + +from rest_framework import serializers +from rest_framework.utils.serializer_helpers import ReturnDict + +from stac_api.models import AssetUpload +from stac_api.models import CollectionAssetUpload +from stac_api.serializers.serializers_utils import NonNullModelSerializer +from stac_api.utils import is_api_version_1 +from stac_api.utils import isoformat +from stac_api.validators import validate_checksum_multihash_sha256 +from stac_api.validators import validate_content_encoding +from stac_api.validators import validate_md5_parts + +logger = logging.getLogger(__name__) + + +class AssetUploadListSerializer(serializers.ListSerializer): + # pylint: disable=abstract-method + + def to_representation(self, data): + return {'uploads': super().to_representation(data)} + + @property + def data(self): + ret = super(serializers.ListSerializer, self).data + return ReturnDict(ret, serializer=self) + + +class UploadPartSerializer(serializers.Serializer): + '''This serializer is used to serialize the data from/to the S3 API. + ''' + # pylint: disable=abstract-method + etag = serializers.CharField(source='ETag', allow_blank=False, required=True) + part_number = serializers.IntegerField( + source='PartNumber', min_value=1, max_value=100, required=True, allow_null=False + ) + modified = serializers.DateTimeField(source='LastModified', required=False, allow_null=True) + size = serializers.IntegerField(source='Size', allow_null=True, required=False) + + +class AssetUploadSerializer(NonNullModelSerializer): + + class Meta: + model = AssetUpload + list_serializer_class = AssetUploadListSerializer + fields = [ + 'upload_id', + 'status', + 'created', + 'checksum_multihash', + 'completed', + 'aborted', + 'number_parts', + 'md5_parts', + 'urls', + 'ended', + 'parts', + 'update_interval', + 'content_encoding' + ] + + checksum_multihash = serializers.CharField( + source='checksum_multihash', + max_length=255, + required=True, + allow_blank=False, + validators=[validate_checksum_multihash_sha256] + ) + md5_parts = serializers.JSONField(required=True) + update_interval = serializers.IntegerField( + required=False, allow_null=False, min_value=-1, max_value=3600, default=-1 + ) + content_encoding = serializers.CharField( + required=False, + allow_null=False, + allow_blank=False, + min_length=1, + max_length=32, + default='', + validators=[validate_content_encoding] + ) + + # write only fields + ended = serializers.DateTimeField(write_only=True, required=False) + parts = serializers.ListField( + child=UploadPartSerializer(), write_only=True, allow_empty=False, required=False + ) + + # Read only fields + upload_id = serializers.CharField(read_only=True) + created = serializers.DateTimeField(read_only=True) + urls = serializers.JSONField(read_only=True) + completed = serializers.SerializerMethodField() + aborted = serializers.SerializerMethodField() + + def validate(self, attrs): + # get partial from kwargs (if partial true and no md5 : ok, if false no md5 : error) + # Check the md5 parts length + if attrs.get('md5_parts') is not None: + validate_md5_parts(attrs['md5_parts'], attrs['number_parts']) + elif not self.partial: + raise serializers.ValidationError( + detail={'md5_parts': _('md5_parts parameter is missing')}, code='missing' + ) + return attrs + + def get_completed(self, obj): + if obj.status == AssetUpload.Status.COMPLETED: + return isoformat(obj.ended) + return None + + def get_aborted(self, obj): + if obj.status == AssetUpload.Status.ABORTED: + return isoformat(obj.ended) + return None + + def get_fields(self): + fields = super().get_fields() + # This is a hack to allow fields with special characters + fields['file:checksum'] = fields.pop('checksum_multihash') + + # Older versions of the api still use different name + request = self.context.get('request') + if not is_api_version_1(request): + fields['checksum:multihash'] = fields.pop('file:checksum') + return fields + + +class AssetUploadPartsSerializer(serializers.Serializer): + '''S3 list_parts response serializer''' + + # pylint: disable=abstract-method + + class Meta: + list_serializer_class = AssetUploadListSerializer + + # Read only fields + parts = serializers.ListField( + source='Parts', child=UploadPartSerializer(), default=list, read_only=True + ) + + +class CollectionAssetUploadSerializer(NonNullModelSerializer): + + class Meta: + model = CollectionAssetUpload + list_serializer_class = AssetUploadListSerializer + fields = [ + 'upload_id', + 'status', + 'created', + 'checksum_multihash', + 'completed', + 'aborted', + 'number_parts', + 'md5_parts', + 'urls', + 'ended', + 'parts', + 'update_interval', + 'content_encoding' + ] + + checksum_multihash = serializers.CharField( + source='checksum_multihash', + max_length=255, + required=True, + allow_blank=False, + validators=[validate_checksum_multihash_sha256] + ) + md5_parts = serializers.JSONField(required=True) + update_interval = serializers.IntegerField( + required=False, allow_null=False, min_value=-1, max_value=3600, default=-1 + ) + content_encoding = serializers.CharField( + required=False, + allow_null=False, + allow_blank=False, + min_length=1, + max_length=32, + default='', + validators=[validate_content_encoding] + ) + + # write only fields + ended = serializers.DateTimeField(write_only=True, required=False) + parts = serializers.ListField( + child=UploadPartSerializer(), write_only=True, allow_empty=False, required=False + ) + + # Read only fields + upload_id = serializers.CharField(read_only=True) + created = serializers.DateTimeField(read_only=True) + urls = serializers.JSONField(read_only=True) + completed = serializers.SerializerMethodField() + aborted = serializers.SerializerMethodField() + + def validate(self, attrs): + # get partial from kwargs (if partial true and no md5 : ok, if false no md5 : error) + # Check the md5 parts length + if attrs.get('md5_parts') is not None: + validate_md5_parts(attrs['md5_parts'], attrs['number_parts']) + elif not self.partial: + raise serializers.ValidationError( + detail={'md5_parts': _('md5_parts parameter is missing')}, code='missing' + ) + return attrs + + def get_completed(self, obj): + if obj.status == CollectionAssetUpload.Status.COMPLETED: + return isoformat(obj.ended) + return None + + def get_aborted(self, obj): + if obj.status == CollectionAssetUpload.Status.ABORTED: + return isoformat(obj.ended) + return None + + def get_fields(self): + fields = super().get_fields() + # This is a hack to allow fields with special characters + fields['file:checksum'] = fields.pop('checksum_multihash') + + # Older versions of the api still use different name + request = self.context.get('request') + if not is_api_version_1(request): + fields['checksum:multihash'] = fields.pop('file:checksum') + return fields diff --git a/app/stac_api/views/upload.py b/app/stac_api/views/upload.py index 20580cf8..7927b6fc 100644 --- a/app/stac_api/views/upload.py +++ b/app/stac_api/views/upload.py @@ -23,9 +23,9 @@ from stac_api.models import CollectionAssetUpload from stac_api.pagination import ExtApiPagination from stac_api.s3_multipart_upload import MultipartUpload -from stac_api.serializers.serializers import AssetUploadPartsSerializer -from stac_api.serializers.serializers import AssetUploadSerializer -from stac_api.serializers.serializers import CollectionAssetUploadSerializer +from stac_api.serializers.upload import AssetUploadPartsSerializer +from stac_api.serializers.upload import AssetUploadSerializer +from stac_api.serializers.upload import CollectionAssetUploadSerializer from stac_api.utils import get_asset_path from stac_api.utils import get_collection_asset_path from stac_api.utils import select_s3_bucket diff --git a/app/tests/tests_09/test_serializer_asset_upload.py b/app/tests/tests_09/test_serializer_asset_upload.py index b1287734..e5a54e43 100644 --- a/app/tests/tests_09/test_serializer_asset_upload.py +++ b/app/tests/tests_09/test_serializer_asset_upload.py @@ -7,7 +7,7 @@ from rest_framework.exceptions import ValidationError from stac_api.models import AssetUpload -from stac_api.serializers.serializers import AssetUploadSerializer +from stac_api.serializers.upload import AssetUploadSerializer from stac_api.utils import get_sha256_multihash from tests.tests_09.base_test import STAC_BASE_V diff --git a/app/tests/tests_10/test_serializer_asset_upload.py b/app/tests/tests_10/test_serializer_asset_upload.py index 9c7c30ab..c5d359a2 100644 --- a/app/tests/tests_10/test_serializer_asset_upload.py +++ b/app/tests/tests_10/test_serializer_asset_upload.py @@ -7,7 +7,7 @@ from rest_framework.exceptions import ValidationError from stac_api.models import AssetUpload -from stac_api.serializers.serializers import AssetUploadSerializer +from stac_api.serializers.upload import AssetUploadSerializer from stac_api.utils import get_sha256_multihash from tests.tests_10.base_test import StacBaseTestCase