diff --git a/mapchete/formats/base.py b/mapchete/formats/base.py index dbface2e..eec86bf6 100644 --- a/mapchete/formats/base.py +++ b/mapchete/formats/base.py @@ -195,8 +195,8 @@ def __init__(self, output_params, readonly=False, **kwargs): ) self.crs = self.pyramid.crs self._bucket = None - self._fs = output_params.get("fs") or None - self._fs_kwargs = output_params.get("fs_kwargs") or {} + self.fs = self._fs = output_params.get("fs") or None + self.fs_kwargs = self._fs_kwargs = output_params.get("fs_kwargs") or {} def is_valid_with_config(self, config): """ diff --git a/mapchete/formats/default/gtiff.py b/mapchete/formats/default/gtiff.py index e8472ba5..5cfe219f 100644 --- a/mapchete/formats/default/gtiff.py +++ b/mapchete/formats/default/gtiff.py @@ -30,33 +30,28 @@ CCITTFAX3, CCITTFAX4, lzma """ -from affine import Affine from contextlib import ExitStack import logging import math -import numpy as np -import numpy.ma as ma import os -import rasterio +import warnings + +from affine import Affine +import numpy as np +from numpy import ma from rasterio.enums import Resampling -from rasterio.io import MemoryFile from rasterio.profiles import Profile from rasterio.rio.overview import get_maximum_overview_level from rasterio.windows import from_bounds from shapely.geometry import box -from tempfile import NamedTemporaryFile from tilematrix import Bounds -import warnings from mapchete.config import validate_values, snap_bounds, _OUTPUT_PARAMETERS from mapchete.errors import MapcheteConfigError from mapchete.formats import base from mapchete.io import ( - fs_from_path, - get_boto3_bucket, makedirs, path_exists, - path_is_remote, ) from mapchete.io.raster import ( write_raster_window, @@ -64,7 +59,6 @@ memory_file, read_raster_no_crs, extract_from_array, - read_raster_window, rasterio_write, ) from mapchete.tile import BufferedTile @@ -258,10 +252,6 @@ def _set_attributes(self, output_params): output_params, nodata=output_params.get("nodata", GTIFF_DEFAULT_PROFILE["nodata"]), ) - self._bucket = ( - self.path.split("/")[2] if self.path.startswith("s3://") else None - ) - self._fs = self.output_params.get("fs") or fs_from_path(self.path) class GTiffTileDirectoryOutputReader( @@ -388,9 +378,6 @@ def write(self, process_tile, data): if data.mask.all(): logger.debug("data empty, nothing to write") else: - # in case of S3 output, create an boto3 resource - bucket_resource = get_boto3_bucket(self._bucket) if self._bucket else None - # Convert from process_tile to output_tiles and write for tile in self.pyramid.intersecting(process_tile): out_path = self.get_path(tile) @@ -403,7 +390,7 @@ def write(self, process_tile, data): out_tile=out_tile, out_path=out_path, tags=tags, - bucket_resource=bucket_resource, + fs=self.fs, ) @@ -424,8 +411,6 @@ def __init__(self, output_params, **kwargs): self.zoom = output_params["delimiters"]["zoom"][0] self.cog = output_params.get("cog", False) self.in_memory = output_params.get("in_memory", True) - _bucket = self.path.split("/")[2] if self.path.startswith("s3://") else None - self._bucket_resource = get_boto3_bucket(_bucket) if _bucket else None def prepare(self, process_area=None, **kwargs): bounds = ( @@ -514,7 +499,7 @@ def prepare(self, process_area=None, **kwargs): logger.debug("open output file: %s", self.path) self._ctx = ExitStack() self.dst = self._ctx.enter_context( - rasterio_write(self.path, "w+", in_memory=self.in_memory, **self._profile) + rasterio_write(self.path, "w+", **self._profile) ) def read(self, output_tile, **kwargs): diff --git a/mapchete/io/raster.py b/mapchete/io/raster.py index 71023729..2cd48643 100644 --- a/mapchete/io/raster.py +++ b/mapchete/io/raster.py @@ -536,7 +536,7 @@ def __init__(self, path, *args, fs=None, in_memory=True, **kwargs): if self.in_memory: self._dst = MemoryFile() else: - self._dst = NamedTemporaryFile() + self._dst = NamedTemporaryFile(suffix=".tif") self._open_args = args self._open_kwargs = kwargs self._sink = None @@ -546,7 +546,7 @@ def __enter__(self): self._sink = self._dst.open(*self._open_args, **self._open_kwargs) else: self._sink = rasterio.open( - self._dst.name, *self._open_args, **self._open_kwargs + self._dst.name, "w+", *self._open_args, **self._open_kwargs ) return self._sink diff --git a/test/test_commands.py b/test/test_commands.py index 86460357..0432718a 100644 --- a/test/test_commands.py +++ b/test/test_commands.py @@ -345,7 +345,11 @@ def test_convert_mapchete(cleantopo_br, mp_tmpdir): assert len(job) job = convert( - cleantopo_br.path, mp_tmpdir, output_pyramid="geodetic", output_metatiling=1 + cleantopo_br.path, + mp_tmpdir, + output_pyramid="geodetic", + output_metatiling=1, + zoom=[1, 4], ) assert len(job) for zoom, row, col in [(4, 15, 31), (3, 7, 15), (2, 3, 7), (1, 1, 3)]: