Merge pull request #588 from ungarj/584_COG_warnings

584 cog warnings

mapchete/commands/_convert.py

@@ -238,10 +238,12 @@ def _empty_callback(*args, **kwargs):
         ),
         config_dir=os.getcwd(),
         zoom_levels=zoom or input_info["zoom_levels"],
-        scale_ratio=scale_ratio,
-        scale_offset=scale_offset,
-        resampling=resampling_method,
-        band_indexes=bidx,
+        process_parameters=dict(
+            scale_ratio=scale_ratio,
+            scale_offset=scale_offset,
+            resampling=resampling_method,
+            band_indexes=bidx,
+        ),
     )
 
     # assert all required information is there
@@ -304,6 +306,8 @@ def _empty_callback(*args, **kwargs):
     mapchete_config.update(
         bounds=(clip_intersection.bounds if clip_geometry else inp_bounds),
         bounds_crs=bounds_crs,
+    )
+    mapchete_config["process_parameters"].update(
         clip_to_output_dtype=mapchete_config["output"].get("dtype", None),
     )
     logger.debug(f"temporary config generated: {pformat(mapchete_config)}")

mapchete/config.py

@@ -91,6 +91,7 @@ class ProcessConfig(BaseModel, arbitrary_types_allowed=True):
     bounds: Union[Tuple[float, float, float, float], None]
     bounds_crs: Union[dict, str, None]
     process_parameters: Union[dict, None]
+    mapchete_file: Union[str, MPath, None]
 
 
 _RESERVED_PARAMETERS = tuple(ProcessConfig.__fields__.keys())

mapchete/formats/default/gtiff.py

@@ -49,7 +49,8 @@
 from mapchete.config import _OUTPUT_PARAMETERS, snap_bounds, validate_values
 from mapchete.errors import MapcheteConfigError
 from mapchete.formats import base
-from mapchete.io import MPath, makedirs, path_exists, path_is_remote
+from mapchete.io import MPath, path_exists, path_is_remote
+from mapchete.io.profiles import DEFAULT_PROFILES
 from mapchete.io.raster import (
     extract_from_array,
     memory_file,
@@ -64,24 +65,7 @@
 logger = logging.getLogger(__name__)
 
 
-class DefaultGTiffProfile(Profile):
-    """Tiled, band-interleaved, DEFLATE-compressed, 8-bit GTiff."""
-
-    defaults = {
-        "driver": "GTiff",
-        "blockysize": 512,
-        "blockxsize": 512,
-        "tiled": True,
-        "dtype": "uint8",
-        "compress": "deflate",
-        "predictor": 2,
-        "interleave": "band",
-        "nodata": 0,
-    }
-
-
 METADATA = {"driver_name": "GTiff", "data_type": "raster", "mode": "rw"}
-GTIFF_DEFAULT_PROFILE = DefaultGTiffProfile()
 IN_MEMORY_THRESHOLD = int(os.environ.get("MP_IN_MEMORY_THRESHOLD", 20000 * 20000))
 
 
@@ -248,7 +232,7 @@ def _set_attributes(self, output_params):
         self.file_extension = ".tif"
         self.output_params = dict(
             output_params,
-            nodata=output_params.get("nodata", GTIFF_DEFAULT_PROFILE["nodata"]),
+            nodata=output_params.get("nodata", DEFAULT_PROFILES["COG"]()["nodata"]),
         )
 
 
@@ -319,7 +303,7 @@ def profile(self, tile=None):
             output profile dictionary used for rasterio.
         """
         dst_metadata = dict(
-            GTIFF_DEFAULT_PROFILE,
+            DEFAULT_PROFILES["COG"](),
             count=self.output_params["bands"],
             **{
                 k: v
@@ -446,8 +430,7 @@ def prepare(self, process_area=None, **kwargs):
         creation_options = {
             k: v for k, v in self.output_params.items() if k not in _OUTPUT_PARAMETERS
         }
-        self._profile = dict(
-            DefaultGTiffProfile(driver="COG" if self.cog else "GTiff"),
+        self._profile = DEFAULT_PROFILES["COG" if self.cog else "GTiff"](
             transform=Affine(
                 self.pyramid.pixel_x_size(self.zoom),
                 0,
@@ -462,13 +445,19 @@ def prepare(self, process_area=None, **kwargs):
             crs=self.pyramid.crs,
             **creation_options,
         )
-        for blocksize in ["blockxsize", "blockysize"]:
-            if self._profile.get(blocksize) is not None:
-                self._profile[blocksize] = int(self._profile[blocksize])
-        logger.debug("single GTiff profile: %s", self._profile)
+        if self.cog:
+            if self._profile.get("blocksize") is not None:
+                self._profile["blocksize"] = int(self._profile.get("blocksize"))
+        else:
+            for blocksize in ["blockxsize", "blockysize"]:
+                if self._profile.get(blocksize) is not None:
+                    self._profile[blocksize] = int(self._profile[blocksize])
+        logger.debug("single GTiff profile: %s", str(self._profile))
         logger.debug(
             get_maximum_overview_level(
-                width, height, minsize=self._profile["blockxsize"]
+                width,
+                height,
+                minsize=self._profile.get("blocksize", self._profile.get("blockxsize")),
             )
         )
         if self.cog or "overviews" in self.output_params:
@@ -483,7 +472,11 @@ def prepare(self, process_area=None, **kwargs):
                     for i in range(
                         1,
                         get_maximum_overview_level(
-                            width, height, minsize=self._profile["blockxsize"]
+                            width,
+                            height,
+                            minsize=self._profile.get(
+                                "blocksize", self._profile.get("blockxsize")
+                            ),
                         ),
                     )
                 ],

mapchete/io/profiles.py

@@ -0,0 +1,31 @@
+from rasterio.profiles import Profile
+
+
+class COGDeflateProfile(Profile):
+    """Standard COG profile."""
+
+    defaults = {
+        "driver": "COG",
+        "blocksize": 512,
+        "compress": "deflate",
+        "predictor": 2,
+        "nodata": 0,
+    }
+
+
+class GTiffDeflateProfile(Profile):
+    """Tiled, band-interleaved, DEFLATE-compressed, 8-bit GTiff."""
+
+    defaults = {
+        "driver": "GTiff",
+        "tiled": True,
+        "blockysize": 512,
+        "blockxsize": 512,
+        "compress": "deflate",
+        "predictor": 2,
+        "interleave": "band",
+        "nodata": 0,
+    }
+
+
+DEFAULT_PROFILES = {"COG": COGDeflateProfile, "GTiff": GTiffDeflateProfile}

test/test_io_raster.py

@@ -12,8 +12,8 @@
 
 import mapchete
 from mapchete.errors import MapcheteIOError
-from mapchete.formats.default.gtiff import DefaultGTiffProfile
 from mapchete.io import path_exists, rasterio_open
+from mapchete.io.profiles import COGDeflateProfile
 from mapchete.io.raster import (
     RasterWindowMemoryFile,
     ReferencedRaster,
@@ -889,7 +889,7 @@ def test_rasterio_write(path, dtype, in_memory):
         width=width,
         height=height,
         crs="EPSG:4326",
-        **DefaultGTiffProfile(dtype=dtype),
+        **COGDeflateProfile(dtype=dtype),
     ) as dst:
         dst.write(arr)
     assert path_exists(path)
@@ -911,7 +911,7 @@ def test_rasterio_write_remote_exception(mp_s3_tmpdir, in_memory):
             width=256,
             height=256,
             crs="EPSG:4326",
-            **DefaultGTiffProfile(dtype="uint8"),
+            **COGDeflateProfile(dtype="uint8"),
         ):
             raise ValueError()