Resolve conflicts

.github/workflows/terraform_build.yaml

@@ -12,8 +12,8 @@ jobs:
 
     - name: Test with pytest
       run: |
-        ./scripts/test
-        ./scripts/test_v2 --no_build
+        ./scripts/test --do-cov --show-warnings
+        ./scripts/test_v2 --no_build --do-cov --show-warnings
 
     - name: Run codacy-coverage-reporter
       uses: codacy/codacy-coverage-reporter-action@master

.github/workflows/terraform_destroy_on_delete.yaml

@@ -4,7 +4,7 @@ on: [delete]
 
 jobs:
   build:
-    if: contains(github.event.ref_type, 'branch') &&  (! github.event.ref == 'master') && (! github.event.ref == 'develop')
+    if: github.event.ref_type == 'branch' && (github.event.ref != 'refs/heads/master') && (github.event.ref != 'refs/heads/develop')
     runs-on: ubuntu-latest
     steps:
     - uses: actions/checkout@v1

.isort.cfg

@@ -2,4 +2,4 @@
 line_length = 88
 multi_line_output = 3
 include_trailing_comma = True
-known_third_party = _pytest,aenum,affine,aiohttp,alembic,asgi_lifespan,async_lru,asyncpg,aws_utils,boto3,botocore,click,docker,errors,fastapi,fiona,gdal_utils,geoalchemy2,geojson,gfw_pixetl,gino,gino_starlette,google,httpx,httpx_auth,logger,logging_utils,moto,numpy,orjson,osgeo,pandas,pendulum,pglast,psutil,psycopg2,pydantic,pyproj,pytest,pytest_asyncio,rasterio,shapely,sqlalchemy,sqlalchemy_utils,starlette,tileputty,typer
+known_third_party = _pytest,aenum,affine,aiohttp,alembic,asgi_lifespan,async_lru,asyncpg,aws_utils,boto3,botocore,click,docker,ee,errors,fastapi,fiona,gdal_utils,geoalchemy2,geojson,gfw_pixetl,gino,gino_starlette,google,httpx,httpx_auth,logger,logging_utils,moto,numpy,orjson,osgeo,pandas,pendulum,pglast,psutil,psycopg2,pydantic,pyproj,pytest,pytest_asyncio,rasterio,shapely,sqlalchemy,sqlalchemy_utils,starlette,tileputty,typer

Dockerfile

@@ -4,12 +4,12 @@ FROM tiangolo/uvicorn-gunicorn-fastapi:python3.10-slim
 ARG ENV
 
 RUN apt-get update -y \
-    && apt-get install --no-install-recommends -y gcc libc-dev musl-dev \
-        postgresql-client libpq-dev make git jq \
+    && apt-get install --no-install-recommends -y gcc g++ libc-dev \
+        postgresql-client libpq-dev make git jq libgdal-dev \
     && apt-get clean \
     && rm -rf /var/lib/apt/lists/*
 
-RUN pip install --upgrade pip && pip install pipenv==v2024.0.1
+RUN pip install --upgrade pip && pip install pipenv==2024.0.1
 #TODO move to pipfile when operational
 RUN pip install newrelic
 

Pipfile

@@ -15,6 +15,7 @@ geopandas = "*"  # Needed by pixetl in batch script test
 gfw_pixetl = {git = "https://github.com/wri/gfw_pixetl.git", ref = "develop"}
 moto = {version = "<5", extras = ["awslambda", "batch", "ec2", "s3", "secretsmanager"]}
 openapi_spec_validator = "*"
+pandas = "<2.2"  # Needed by pixetl in batch script test
 pre-commit = "*"
 pytest = "*"
 pytest-asyncio = "*"
@@ -44,7 +45,7 @@ google-cloud-storage = "*"
 httpcore = "*"
 httpx = "*"
 httpx-auth = "*"
-numpy = "*"
+numpy = "<2.0"
 orjson = "*"
 packaging = "*"
 pendulum = "<3"

app/authentication/api_keys.py

@@ -93,32 +93,14 @@ def api_key_is_valid(
     return is_valid
 
 
-def api_key_is_internal(
-    domains: List[str],
-    user_id: Optional[str] = None,
-    origin: Optional[str] = None,
-    referrer: Optional[str] = None,
-) -> bool:
-
-    is_internal: bool = False
-    if origin and domains:
-        is_internal = any(
-            [
-                re.search(_to_regex(internal_domain.strip()), domain)
-                for domain in domains
-                for internal_domain in INTERNAL_DOMAINS.split(",")
-            ]
-        )
-    elif referrer and domains:
-        is_internal = any(
-            [
-                re.search(_to_regex(domain), internal_domain)
-                for domain in domains
-                for internal_domain in INTERNAL_DOMAINS.split(",")
-            ]
-        )
-
-    return is_internal
+def api_key_is_internal(domains: List[str]) -> bool:
+    return any(
+        [
+            re.search(_to_regex(internal_domain.strip()), domain)
+            for domain in domains
+            for internal_domain in INTERNAL_DOMAINS.split(",")
+        ]
+    )
 
 
 def _api_key_origin_auto_error(
@@ -139,7 +121,7 @@ def _api_key_origin_auto_error(
 
 def _to_regex(domain):
     result = domain.replace(".", r"\.").replace("*", ".*")
-    return fr"^{result}$"
+    return rf"^{result}$"
 
 
 def _extract_domain(url: str) -> str:

app/crud/assets.py

@@ -3,23 +3,18 @@
 
 from asyncpg import UniqueViolationError
 from fastapi.encoders import jsonable_encoder
-from sqlalchemy.sql import and_
+from sqlalchemy import func
 
 from app.crud.metadata import (
     create_asset_metadata,
     get_asset_metadata,
     update_asset_metadata,
 )
-from sqlalchemy import func
-from async_lru import alru_cache
-
 from ..errors import RecordAlreadyExistsError, RecordNotFoundError
-from ..models.enum.assets import AssetType
 from ..models.orm.asset_metadata import AssetMetadata as ORMAssetMetadata
 from ..models.orm.assets import Asset as ORMAsset
 from ..models.orm.versions import Version as ORMVersion
 from ..models.pydantic.creation_options import CreationOptions, creation_option_factory
-from ..models.pydantic.asset_metadata import RasterTileSetMetadataOut
 from . import update_data, versions
 
 
@@ -211,7 +206,7 @@ async def create_asset(dataset, version, **data) -> ORMAsset:
     except UniqueViolationError:
         raise RecordAlreadyExistsError(
             f"Cannot create asset of type {data['asset_type']}. "
-            f"Asset uri must be unique. An asset with uri {data['asset_uri']} already exists"
+            f"Asset URI must be unique. An asset with URI {data['asset_uri']} already exists"
         )
 
     if metadata_data:

app/crud/versions.py

@@ -1,5 +1,6 @@
 from typing import Any, Dict, List, Optional
 
+from async_lru import alru_cache
 from asyncpg import UniqueViolationError
 
 from ..errors import RecordAlreadyExistsError, RecordNotFoundError
@@ -11,7 +12,6 @@
 from . import datasets, update_data
 from .metadata import (
     create_version_metadata,
-    update_all_metadata,
     update_version_metadata,
 )
 
@@ -52,8 +52,12 @@ async def get_version(dataset: str, version: str) -> ORMVersion:
     return row
 
 
+@alru_cache(maxsize=64, ttl=15.0)
 async def get_latest_version(dataset) -> str:
     """Fetch latest version number."""
+    # NOTE: Even though we invalidate cache entries in the known mutator
+    # functions, set cache TTL to a low value to reduce incoherency between
+    # ECS instances.
 
     latest: Optional[str] = (
         await ORMVersion.select("version")
@@ -80,9 +84,6 @@ async def create_version(dataset: str, version: str, **data) -> ORMVersion:
     if data.get("is_downloadable") is None:
         data["is_downloadable"] = d.is_downloadable
 
-    if data.get("is_latest"):
-        await _reset_is_latest(dataset, version)
-
     metadata_data = data.pop("metadata", None)
     try:
         new_version: ORMVersion = await ORMVersion.create(
@@ -100,6 +101,13 @@ async def create_version(dataset: str, version: str, **data) -> ORMVersion:
         )
         new_version.metadata = metadata
 
+    # NOTE: We disallow specifying a new version as latest on creation via
+    # the VersionCreateIn model in order to prevent requests temporarily going
+    # to an incompletely-imported asset, however it's technically allowed in
+    # this function to facilitate testing.
+    if data.get("is_latest"):
+        await _reset_is_latest(dataset, version)
+
     return new_version
 
 
@@ -155,8 +163,15 @@ async def _update_is_downloadable(
 
 
 async def _reset_is_latest(dataset: str, version: str) -> None:
+    """Set is_latest to False for all other versions of a dataset."""
+    # NOTE: Please remember to only call after setting the provided version to
+    # latest to avoid setting nothing to latest
+    # FIXME: This will get slower and more DB-intensive the more versions
+    # there are for a dataset. Could be re-written to use a single DB call,
+    # no?
     versions = await get_versions(dataset)
     version_gen = list_to_async_generator(versions)
     async for version_orm in version_gen:
         if version_orm.version != version:
             await update_version(dataset, version_orm.version, is_latest=False)
+    _: bool = get_latest_version.cache_invalidate(dataset)

app/models/enum/assets.py

@@ -23,6 +23,7 @@ class AssetType(str, Enum):
     csv = "csv"
     tsv = "tsv"
     grid_1x1 = "1x1 grid"
+    cog = "COG"
     # esri_map_service = "ESRI Map Service"
     # esri_feature_service = "ESRI Feature Service"
     # esri_image_service = "ESRI Image Service"
@@ -63,6 +64,7 @@ def is_single_file_asset(asset_type: str) -> bool:
         AssetType.csv,
         AssetType.tsv,
         AssetType.grid_1x1,
+        AssetType.cog,
     ]
 
 

app/models/enum/creation_options.py

@@ -89,3 +89,14 @@ class ColorMapType(str, Enum):
     date_conf_intensity_multi_16 = "date_conf_intensity_multi_16"
     year_intensity = "year_intensity"
     value_intensity = "value_intensity"
+
+
+class TileBlockSize(int, Enum):
+    two_fifty_six = 256
+    five_twelve = 512
+    ten_twenty_four = 1024
+
+
+class Srid(str, Enum):
+    wgs84 = "epsg:4326"
+    web_mercator = "epsg:3857"

app/models/pydantic/asset_metadata.py

@@ -6,7 +6,9 @@
 
 from ...models.orm.assets import Asset as ORMAsset
 from ..enum.assets import AssetType
+from ..enum.creation_options import TileBlockSize
 from ..enum.pg_types import PGType
+from ..enum.pixetl import ResamplingMethod
 from .base import BaseORMRecord, StrictBaseModel
 from .responses import Response
 
@@ -75,6 +77,11 @@ class RasterTileSetMetadata(AssetBase):
     resolution: Optional[int]
 
 
+class COGMetadata(AssetBase):
+    block_size: TileBlockSize
+    resampling: ResamplingMethod
+
+
 class RasterTileSetMetadataUpdate(AssetBase):
     resolution: int
 
@@ -186,6 +193,7 @@ def asset_metadata_factory(asset: ORMAsset) -> AssetMetadata:
         AssetType.grid_1x1: VectorFileMetadata,
         AssetType.shapefile: VectorFileMetadata,
         AssetType.geopackage: VectorFileMetadata,
+        AssetType.cog: COGMetadata,
     }
 
     if asset.asset_type in metadata_factory.keys():
@@ -235,5 +243,6 @@ class FieldsMetadataResponse(Response):
 class FieldMetadataResponse(Response):
     data: FieldMetadataOut
 
+
 class RasterBandsMetadataResponse(Response):
     data: List[RasterBandMetadata]

app/models/pydantic/authentication.py

@@ -16,7 +16,7 @@ class SignUpRequestIn(StrictBaseModel):
 
 class User(BaseModel):
     id: str
-    name: str
+    name: Optional[str]
     email: EmailStr
     createdAt: datetime
     role: str

app/models/pydantic/creation_options.py

@@ -14,6 +14,7 @@
     PartitionType,
     RasterDrivers,
     TableDrivers,
+    TileBlockSize,
     TileStrategy,
     VectorDrivers,
 )
@@ -225,10 +226,8 @@ class VectorSourceCreationOptions(StrictBaseModel):
     def validate_source_uri(cls, v, values, **kwargs):
         if values.get("source_driver") == VectorDrivers.csv:
             assert len(v) >= 1, "CSV sources require at least one input file"
-        else:
-            assert (
-                len(v) == 1
-            ), "Non-CSV vector sources require one and only one input file"
+        elif values.get("source_driver") in [VectorDrivers.esrijson, VectorDrivers.shp, VectorDrivers.geojson_seq, VectorDrivers.geojson]:
+            assert (len(v) == 1), "GeoJSON and ESRI Shapefile vector sources require one and only one input file"
         return v
 
 
@@ -339,6 +338,36 @@ class RasterTileCacheCreationOptions(TileCacheBaseModel):
     )
 
 
+class COGCreationOptions(StrictBaseModel):
+    implementation: str = Field(
+        "default",
+        description="Name space to use for COG. "
+        "This will be part of the URI and will "
+        "allow creation of multiple COGs per version.",
+    )
+    source_asset_id: str = Field(
+        ...,
+        description="Raster tile set asset ID to use as source. "
+        "Must be an asset of the same version",
+    )
+    resampling: ResamplingMethod = Field(
+        ResamplingMethod.average,
+        description="Resampling method used to downsample overviews",
+    )
+    block_size: Optional[TileBlockSize] = Field(
+        512,
+        description="Block size to tile COG with.",
+    )
+    compute_stats: bool = False
+    export_to_gee: bool = Field(
+        False,
+        description="Option to export COG to a Google Cloud Storage and create"
+        " a COG-backed asset on Google Earth Engine (GEE). The asset will be created"
+        " under the project `forma-250` with the asset ID `{dataset}/{implementation}. "
+        "Versioning is currently not supported due to GEE storage constraints.",
+    )
+
+
 class DynamicVectorTileCacheCreationOptions(TileCacheBaseModel):
     field_attributes: Optional[List[Dict[str, Any]]] = Field(
         None,
@@ -382,8 +411,7 @@ class StaticVectorFileCreationOptions(StrictBaseModel):
 
 class StaticVector1x1CreationOptions(StaticVectorFileCreationOptions):
     include_tile_id: Optional[bool] = Field(
-        False,
-        description="Whether or not to include the tile_id of each feature"
+        False, description="Whether or not to include the tile_id of each feature"
     )
 
 
@@ -395,6 +423,7 @@ class StaticVector1x1CreationOptions(StaticVectorFileCreationOptions):
 
 OtherCreationOptions = Union[
     TableAssetCreationOptions,
+    COGCreationOptions,
     RasterTileCacheCreationOptions,
     StaticVectorTileCacheCreationOptions,
     StaticVectorFileCreationOptions,
@@ -424,6 +453,7 @@ class CreationOptionsResponse(Response):
     AssetType.shapefile: StaticVectorFileCreationOptions,
     AssetType.geopackage: StaticVectorFileCreationOptions,
     AssetType.raster_tile_set: RasterTileSetAssetCreationOptions,
+    AssetType.cog: COGCreationOptions,
     AssetType.raster_tile_cache: RasterTileCacheCreationOptions,
     AssetType.database_table: TableAssetCreationOptions,
 }