clean up code; use self.fs instead of self._bucket_resource

mapchete/formats/base.py

@@ -195,8 +195,8 @@ def __init__(self, output_params, readonly=False, **kwargs):
         )
         self.crs = self.pyramid.crs
         self._bucket = None
-        self._fs = output_params.get("fs") or None
-        self._fs_kwargs = output_params.get("fs_kwargs") or {}
+        self.fs = self._fs = output_params.get("fs") or None
+        self.fs_kwargs = self._fs_kwargs = output_params.get("fs_kwargs") or {}
 
     def is_valid_with_config(self, config):
         """

mapchete/formats/default/gtiff.py

@@ -30,41 +30,35 @@
     CCITTFAX3, CCITTFAX4, lzma
 """
 
-from affine import Affine
 from contextlib import ExitStack
 import logging
 import math
-import numpy as np
-import numpy.ma as ma
 import os
-import rasterio
+import warnings
+
+from affine import Affine
+import numpy as np
+from numpy import ma
 from rasterio.enums import Resampling
-from rasterio.io import MemoryFile
 from rasterio.profiles import Profile
 from rasterio.rio.overview import get_maximum_overview_level
 from rasterio.windows import from_bounds
 from shapely.geometry import box
-from tempfile import NamedTemporaryFile
 from tilematrix import Bounds
-import warnings
 
 from mapchete.config import validate_values, snap_bounds, _OUTPUT_PARAMETERS
 from mapchete.errors import MapcheteConfigError
 from mapchete.formats import base
 from mapchete.io import (
-    fs_from_path,
-    get_boto3_bucket,
     makedirs,
     path_exists,
-    path_is_remote,
 )
 from mapchete.io.raster import (
     write_raster_window,
     prepare_array,
     memory_file,
     read_raster_no_crs,
     extract_from_array,
-    read_raster_window,
     rasterio_write,
 )
 from mapchete.tile import BufferedTile
@@ -258,10 +252,6 @@ def _set_attributes(self, output_params):
             output_params,
             nodata=output_params.get("nodata", GTIFF_DEFAULT_PROFILE["nodata"]),
         )
-        self._bucket = (
-            self.path.split("/")[2] if self.path.startswith("s3://") else None
-        )
-        self._fs = self.output_params.get("fs") or fs_from_path(self.path)
 
 
 class GTiffTileDirectoryOutputReader(
@@ -388,9 +378,6 @@ def write(self, process_tile, data):
         if data.mask.all():
             logger.debug("data empty, nothing to write")
         else:
-            # in case of S3 output, create an boto3 resource
-            bucket_resource = get_boto3_bucket(self._bucket) if self._bucket else None
-
             # Convert from process_tile to output_tiles and write
             for tile in self.pyramid.intersecting(process_tile):
                 out_path = self.get_path(tile)
@@ -403,7 +390,7 @@ def write(self, process_tile, data):
                     out_tile=out_tile,
                     out_path=out_path,
                     tags=tags,
-                    bucket_resource=bucket_resource,
+                    fs=self.fs,
                 )
 
 
@@ -424,8 +411,6 @@ def __init__(self, output_params, **kwargs):
         self.zoom = output_params["delimiters"]["zoom"][0]
         self.cog = output_params.get("cog", False)
         self.in_memory = output_params.get("in_memory", True)
-        _bucket = self.path.split("/")[2] if self.path.startswith("s3://") else None
-        self._bucket_resource = get_boto3_bucket(_bucket) if _bucket else None
 
     def prepare(self, process_area=None, **kwargs):
         bounds = (
@@ -514,7 +499,7 @@ def prepare(self, process_area=None, **kwargs):
         logger.debug("open output file: %s", self.path)
         self._ctx = ExitStack()
         self.dst = self._ctx.enter_context(
-            rasterio_write(self.path, "w+", in_memory=self.in_memory, **self._profile)
+            rasterio_write(self.path, "w+", **self._profile)
         )
 
     def read(self, output_tile, **kwargs):

mapchete/io/raster.py

@@ -536,7 +536,7 @@ def __init__(self, path, *args, fs=None, in_memory=True, **kwargs):
         if self.in_memory:
             self._dst = MemoryFile()
         else:
-            self._dst = NamedTemporaryFile()
+            self._dst = NamedTemporaryFile(suffix=".tif")
         self._open_args = args
         self._open_kwargs = kwargs
         self._sink = None
@@ -546,7 +546,7 @@ def __enter__(self):
             self._sink = self._dst.open(*self._open_args, **self._open_kwargs)
         else:
             self._sink = rasterio.open(
-                self._dst.name, *self._open_args, **self._open_kwargs
+                self._dst.name, "w+", *self._open_args, **self._open_kwargs
             )
         return self._sink
 

test/test_commands.py

@@ -345,7 +345,11 @@ def test_convert_mapchete(cleantopo_br, mp_tmpdir):
     assert len(job)
 
     job = convert(
-        cleantopo_br.path, mp_tmpdir, output_pyramid="geodetic", output_metatiling=1
+        cleantopo_br.path,
+        mp_tmpdir,
+        output_pyramid="geodetic",
+        output_metatiling=1,
+        zoom=[1, 4],
     )
     assert len(job)
     for zoom, row, col in [(4, 15, 31), (3, 7, 15), (2, 3, 7), (1, 1, 3)]: