From 6e84f7678dacf445b8ddd8e11e4f26adce5443cb Mon Sep 17 00:00:00 2001 From: Joachim Ungar Date: Thu, 8 Jul 2021 13:22:21 +0200 Subject: [PATCH 1/9] add ResultGenerator class --- mapchete/__init__.py | 4 ++-- mapchete/_processing.py | 16 ++++++++++++++++ 2 files changed, 18 insertions(+), 2 deletions(-) diff --git a/mapchete/__init__.py b/mapchete/__init__.py index eaca058b..f2261228 100644 --- a/mapchete/__init__.py +++ b/mapchete/__init__.py @@ -1,12 +1,12 @@ import logging from mapchete._core import open, Mapchete -from mapchete._processing import MapcheteProcess +from mapchete._processing import MapcheteProcess, ProcessInfo from mapchete.tile import count_tiles from mapchete._timer import Timer -__all__ = ["open", "count_tiles", "Mapchete", "MapcheteProcess", "Timer"] +__all__ = ["open", "count_tiles", "Mapchete", "MapcheteProcess", "ProcessInfo", "Timer"] __version__ = "0.40" diff --git a/mapchete/_processing.py b/mapchete/_processing.py index dd51d967..cb05f2af 100644 --- a/mapchete/_processing.py +++ b/mapchete/_processing.py @@ -21,6 +21,22 @@ ProcessInfo = namedtuple("ProcessInfo", "tile processed process_msg written write_msg") +class ResultGenerator: + """ + Class to encapsulate processing generator while exposing generator length. + """ + + def __init__(self, generator, length): + self._generator = generator + self._length = length + + def __len__(self): + return self._length + + def __iter__(self): + return self._generator + + class TileProcess: """ Class to process on a specific process tile. From 85911ee3cdb423397b7a4b272355f4d83b6ae8ae Mon Sep 17 00:00:00 2001 From: Joachim Ungar Date: Thu, 15 Jul 2021 20:41:53 +0200 Subject: [PATCH 2/9] add cp command --- mapchete/_core.py | 4 +- mapchete/cli/default/cp.py | 161 ++++-------------------- mapchete/cli/utils.py | 8 +- mapchete/commands/__init__.py | 8 ++ mapchete/commands/_cp.py | 223 ++++++++++++++++++++++++++++++++++ mapchete/commands/_job.py | 33 +++++ mapchete/config.py | 8 +- test/test_commands.py | 43 +++++++ 8 files changed, 344 insertions(+), 144 deletions(-) create mode 100644 mapchete/commands/__init__.py create mode 100644 mapchete/commands/_cp.py create mode 100644 mapchete/commands/_job.py create mode 100644 test/test_commands.py diff --git a/mapchete/_core.py b/mapchete/_core.py index 7d4719fc..ed7aef5c 100644 --- a/mapchete/_core.py +++ b/mapchete/_core.py @@ -291,7 +291,7 @@ def batch_processor( ): yield process_info - def count_tiles(self, minzoom, maxzoom, init_zoom=0): + def count_tiles(self, minzoom=None, maxzoom=None, init_zoom=0): """ Count number of tiles intersecting with process area at given zoom levels. @@ -307,6 +307,8 @@ def count_tiles(self, minzoom, maxzoom, init_zoom=0): ------- number of tiles """ + minzoom = min(self.config.init_zoom_levels) if minzoom is None else minzoom + maxzoom = max(self.config.init_zoom_levels) if maxzoom is None else maxzoom if (minzoom, maxzoom) not in self._count_tiles_cache: self._count_tiles_cache[(minzoom, maxzoom)] = count_tiles( self.config.area_at_zoom(), diff --git a/mapchete/cli/default/cp.py b/mapchete/cli/default/cp.py index 98539923..15060344 100644 --- a/mapchete/cli/default/cp.py +++ b/mapchete/cli/default/cp.py @@ -1,32 +1,18 @@ import click -import fiona -import json -import logging -import os -from shapely.geometry import box, shape -from shapely.ops import unary_union -from tilematrix import TilePyramid import tqdm -import mapchete +from mapchete import commands from mapchete.cli import utils -from mapchete.config import _guess_geometry -from mapchete.formats import read_output_metadata -from mapchete.io import fs_from_path, tiles_exist -from mapchete.io.vector import reproject_geometry - -logger = logging.getLogger(__name__) @click.command(help="Copy TileDirectory from one source to another.") -@utils.arg_input -@utils.arg_output +@utils.arg_src_tiledir +@utils.arg_dst_tiledir @utils.opt_zoom @utils.opt_area @utils.opt_area_crs @utils.opt_bounds @utils.opt_bounds_crs -@utils.opt_wkt_geometry @utils.opt_overwrite @utils.opt_verbose @utils.opt_no_pbar @@ -37,125 +23,24 @@ @utils.opt_http_password @utils.opt_src_fs_opts @utils.opt_dst_fs_opts -def cp( - input_, - output, - zoom=None, - area=None, - area_crs=None, - bounds=None, - bounds_crs=None, - wkt_geometry=None, - overwrite=False, - verbose=False, - no_pbar=False, - debug=False, - logfile=None, - multi=None, - username=None, - password=None, - src_fs_opts=None, - dst_fs_opts=None, -): +def cp(*args, debug=False, no_pbar=False, verbose=False, logfile=None, **kwargs): """Copy TileDirectory.""" - if zoom is None: # pragma: no cover - raise click.UsageError("zoom level(s) required") - - src_fs = fs_from_path(input_, username=username, password=password, **src_fs_opts) - dst_fs = fs_from_path(output, username=username, password=password, **dst_fs_opts) - - # open source tile directory - with mapchete.open( - input_, - zoom=zoom, - area=area, - area_crs=area_crs, - bounds=bounds, - bounds_crs=bounds_crs, - wkt_geometry=wkt_geometry, - username=username, - password=password, - fs=src_fs, - fs_kwargs=src_fs_opts, - ) as src_mp: - tp = src_mp.config.output_pyramid - - # copy metadata to destination if necessary - src_metadata = os.path.join(input_, "metadata.json") - dst_metadata = os.path.join(output, "metadata.json") - if not dst_fs.exists(dst_metadata): - logger.debug(f"copy {src_metadata} to {dst_metadata}") - _copy(src_fs, src_metadata, dst_fs, dst_metadata) - - with mapchete.open( - output, - zoom=zoom, - area=area, - area_crs=area_crs, - bounds=bounds, - bounds_crs=bounds_crs, - wkt_geometry=wkt_geometry, - username=username, - password=password, - fs=dst_fs, - fs_kwargs=dst_fs_opts, - ) as dst_mp: - for z in range(min(zoom), max(zoom) + 1): - click.echo(f"copy zoom {z}...") - # materialize all tiles - aoi_geom = src_mp.config.area_at_zoom(z) - tiles = [ - t - for t in tp.tiles_from_geom(aoi_geom, z) - # this is required to omit tiles touching the config area - if aoi_geom.intersection(t.bbox).area - ] - - # check which source tiles exist - logger.debug("looking for existing source tiles...") - src_tiles_exist = { - tile: exists - for tile, exists in tiles_exist( - config=src_mp.config, output_tiles=tiles, multi=multi - ) - } - - logger.debug("looking for existing destination tiles...") - # chech which destination tiles exist - dst_tiles_exist = { - tile: exists - for tile, exists in tiles_exist( - config=dst_mp.config, output_tiles=tiles, multi=multi - ) - } - - # copy - for tile in tqdm.tqdm(tiles, unit="tile", disable=debug or no_pbar): - src_path = src_mp.config.output_reader.get_path(tile) - # only copy if source tile exists - if src_tiles_exist[tile]: - # skip if destination tile exists and overwrite is deactivated - if dst_tiles_exist[tile] and not overwrite: - logger.debug(f"{tile}: destination tile exists") - continue - # copy from source to target - else: - dst_path = dst_mp.config.output_reader.get_path(tile) - logger.debug(f"{tile}: copy {src_path} to {dst_path}") - _copy(src_fs, src_path, dst_fs, dst_path) - else: - logger.debug(f"{tile}: source tile ({src_path}) does not exist") - - -def _copy(src_fs, src_path, dst_fs, dst_path): - # create parent directories on local filesystems - if dst_fs.protocol == "file": - dst_fs.mkdir(os.path.dirname(dst_path), create_parents=True) - - # copy either within a filesystem or between filesystems - if src_fs == dst_fs: - src_fs.copy(src_path, dst_path) - else: - with src_fs.open(src_path, "rb") as src: - with dst_fs.open(dst_path, "wb") as dst: - dst.write(src.read()) + for x in ["password", "username"]: + if kwargs.get(x): # pragma: no cover + raise click.BadOptionUsage( + x, + f"'--{x} foo' is deprecated. You should use '--src-fs-opts {x}=foo' instead.", + ) + kwargs.pop(x) + list( + tqdm.tqdm( + commands.cp( + *args, + as_iterator=True, + msg_callback=tqdm.tqdm.write if verbose else None, + **kwargs, + ), + unit="tile", + disable=debug or no_pbar, + ) + ) diff --git a/mapchete/cli/utils.py b/mapchete/cli/utils.py index 7e5ebd14..6d648f88 100644 --- a/mapchete/cli/utils.py +++ b/mapchete/cli/utils.py @@ -153,6 +153,8 @@ def _cb_key_val(ctx, param, value): "inputs", metavar="INPUTS", nargs=-1, callback=_validate_inputs ) arg_output = click.argument("output", type=click.STRING) +arg_src_tiledir = click.argument("src_tiledir", type=click.STRING) +arg_dst_tiledir = click.argument("dst_tiledir", type=click.STRING) # click options # @@ -356,21 +358,21 @@ def _cb_key_val(ctx, param, value): metavar="NAME=VALUE", multiple=True, callback=_cb_key_val, - help="Configuration options for source fsspec filesystem. ", + help="Configuration options for source fsspec filesystem.", ) opt_dst_fs_opts = click.option( "--dst-fs-opts", metavar="NAME=VALUE", multiple=True, callback=_cb_key_val, - help="Configuration options for destination fsspec filesystem. ", + help="Configuration options for destination fsspec filesystem.", ) opt_fs_opts = click.option( "--fs-opts", metavar="NAME=VALUE", multiple=True, callback=_cb_key_val, - help="Configuration options for destination fsspec filesystem. ", + help="Configuration options for destination fsspec filesystem.", ) diff --git a/mapchete/commands/__init__.py b/mapchete/commands/__init__.py new file mode 100644 index 00000000..ac394213 --- /dev/null +++ b/mapchete/commands/__init__.py @@ -0,0 +1,8 @@ +""" +This package contains easy to access functions which otherwise would have to be called via the CLI. +This should make the use from within other scripts, notebooks, etc. easier. +""" +from mapchete.commands._cp import cp + + +__all__ = ["cp"] diff --git a/mapchete/commands/_cp.py b/mapchete/commands/_cp.py new file mode 100644 index 00000000..51c8a59d --- /dev/null +++ b/mapchete/commands/_cp.py @@ -0,0 +1,223 @@ +import fiona +import json +import logging +import os +from rasterio.crs import CRS +from shapely.geometry import box, shape +from shapely.geometry.base import BaseGeometry +from shapely.ops import unary_union +from tilematrix import TilePyramid +import tqdm +from typing import Callable, List, Tuple, Union + +import mapchete +from mapchete.commands._job import empty_callback, Job +from mapchete.config import _guess_geometry +from mapchete.formats import read_output_metadata +from mapchete.io import fs_from_path, tiles_exist +from mapchete.io.vector import reproject_geometry + +logger = logging.getLogger(__name__) + + +def cp( + src_tiledir: str, + dst_tiledir: str, + zoom: Union[int, List[int]] = None, + area: Union[BaseGeometry, str, dict] = None, + area_crs: str = None, + bounds: Tuple[float] = None, + bounds_crs: Tuple[CRS, str] = None, + overwrite: bool = False, + multi: int = None, + src_fs_opts: dict = None, + dst_fs_opts: dict = None, + msg_callback: Callable = None, + as_iterator: bool = False, +) -> Job: + """ + Copy TileDirectory from source to destination. + + Parameters + ---------- + src_tiledir : str + Source TileDirectory or mapchete file. + dst_tiledir : str + Destination TileDirectory. + zoom : integer or list of integers + Single zoom, minimum and maximum zoom or a list of zoom levels. + area : str, dict, BaseGeometry + Geometry to override bounds or area provided in process configuration. Can be either a + WKT string, a GeoJSON mapping, a shapely geometry or a path to a Fiona-readable file. + area_crs : CRS or str + CRS of area (default: process CRS). + bounds : tuple + Override bounds or area provided in process configuration. + bounds_crs : CRS or str + CRS of area (default: process CRS). + overwrite : bool + Overwrite existing output. + multi : int + Number of threads used to check whether tiles exist. + src_fs_opts : dict + Configuration options for source fsspec filesystem. + dst_fs_opts : dict + Configuration options for destination fsspec filesystem. + msg_callback : Callable + Optional callback function for process messages. + as_iterator : bool + Returns as generator but with a __len__() property. + + Returns + ------- + Job instance either with already processed items or a generator with known length. + + Examples + -------- + >>> cp("foo", "bar") + + This will run the whole copy process. + + >>> for i in cp("foo", "bar", as_iterator=True): + >>> print(i) + + This will return a generator where through iteration, tiles are copied. + + >>> list(tqdm.tqdm(cp("foo", "bar", as_iterator=True))) + + Usage within a process bar. + """ + msg_callback = msg_callback or empty_callback + src_fs_opts = src_fs_opts or {} + dst_fs_opts = dst_fs_opts or {} + if zoom is None: # pragma: no cover + raise ValueError("zoom level(s) required") + + src_fs = fs_from_path(src_tiledir, **src_fs_opts) + dst_fs = fs_from_path(dst_tiledir, **dst_fs_opts) + + # open source tile directory + with mapchete.open( + src_tiledir, + zoom=zoom, + area=area, + area_crs=area_crs, + bounds=bounds, + bounds_crs=bounds_crs, + fs=src_fs, + fs_kwargs=src_fs_opts, + ) as src_mp: + tp = src_mp.config.output_pyramid + + # copy metadata to destination if necessary + src_metadata = os.path.join(src_tiledir, "metadata.json") + dst_metadata = os.path.join(dst_tiledir, "metadata.json") + if not dst_fs.exists(dst_metadata): + msg = f"copy {src_metadata} to {dst_metadata}" + logger.debug(msg) + msg_callback(msg) + _copy(src_fs, src_metadata, dst_fs, dst_metadata) + + with mapchete.open( + dst_tiledir, + zoom=zoom, + area=area, + area_crs=area_crs, + bounds=bounds, + bounds_crs=bounds_crs, + fs=dst_fs, + fs_kwargs=dst_fs_opts, + ) as dst_mp: + return Job( + _copy_tiles, + msg_callback, + src_mp, + dst_mp, + tp, + multi, + src_fs, + dst_fs, + overwrite, + as_iterator=as_iterator, + total=src_mp.count_tiles(), + ) + + +def _copy_tiles( + msg_callback, + src_mp, + dst_mp, + tp, + multi, + src_fs, + dst_fs, + overwrite, +): + for z in src_mp.config.init_zoom_levels: + msg_callback(f"copy tiles for zoom {z}...") + # materialize all tiles + aoi_geom = src_mp.config.area_at_zoom(z) + tiles = [ + t + for t in tp.tiles_from_geom(aoi_geom, z) + # this is required to omit tiles touching the config area + if aoi_geom.intersection(t.bbox).area + ] + + # check which source tiles exist + logger.debug("looking for existing source tiles...") + src_tiles_exist = { + tile: exists + for tile, exists in tiles_exist( + config=src_mp.config, output_tiles=tiles, multi=multi + ) + } + + logger.debug("looking for existing destination tiles...") + # chech which destination tiles exist + dst_tiles_exist = { + tile: exists + for tile, exists in tiles_exist( + config=dst_mp.config, output_tiles=tiles, multi=multi + ) + } + + # copy + copied = 0 + for tile in tiles: + src_path = src_mp.config.output_reader.get_path(tile) + # only copy if source tile exists + if src_tiles_exist[tile]: + # skip if destination tile exists and overwrite is deactivated + if dst_tiles_exist[tile] and not overwrite: + msg = f"{tile}: destination tile exists" + logger.debug(msg) + yield msg + continue + # copy from source to target + else: + dst_path = dst_mp.config.output_reader.get_path(tile) + _copy(src_fs, src_path, dst_fs, dst_path) + copied += 1 + msg = f"{tile}: copy {src_path} to {dst_path}" + logger.debug(msg) + yield msg + else: + msg = f"{tile}: source tile ({src_path}) does not exist" + logger.debug(msg) + yield msg + msg_callback(f"{copied} tiles copied") + + +def _copy(src_fs, src_path, dst_fs, dst_path): + # create parent directories on local filesystems + if dst_fs.protocol == "file": + dst_fs.makedirs(os.path.dirname(dst_path), exist_ok=True) + + # copy either within a filesystem or between filesystems + if src_fs == dst_fs: + src_fs.copy(src_path, dst_path) + else: + with src_fs.open(src_path, "rb") as src: + with dst_fs.open(dst_path, "wb") as dst: + dst.write(src.read()) diff --git a/mapchete/commands/_job.py b/mapchete/commands/_job.py new file mode 100644 index 00000000..0b33a4aa --- /dev/null +++ b/mapchete/commands/_job.py @@ -0,0 +1,33 @@ +from typing import Generator + + +class Job: + """Wraps the output of a processing function into a generator with known length.""" + + def __init__( + self, + func: Generator, + *fargs: dict, + as_iterator: bool = False, + total: int = None, + **fkwargs: dict + ): + self.func = func + self.fargs = fargs + self.fkwargs = fkwargs + self._total = total + self._as_iterator = as_iterator + if not as_iterator: + list(self.func(*self.fargs, **self.fkwargs)) + + def __len__(self): + return self._total + + def __iter__(self): + if not self._as_iterator: + raise TypeError("initialize with 'as_iterator=True'") + return self.func(*self.fargs, **self.fkwargs) + + +def empty_callback(*args, **kwargs): + pass diff --git a/mapchete/config.py b/mapchete/config.py index 4b360670..dfcaf997 100644 --- a/mapchete/config.py +++ b/mapchete/config.py @@ -590,7 +590,9 @@ def params_at_zoom(self, zoom): zoom level dependent process configuration """ if zoom not in self.init_zoom_levels: - raise ValueError("zoom level not available with current configuration") + raise ValueError( + f"zoom level {zoom} not available with current configuration: {self.init_zoom_levels}" + ) out = OrderedDict( self._params_at_zoom[zoom], input=OrderedDict(), output=self.output ) @@ -630,7 +632,9 @@ def area_at_zoom(self, zoom=None): return self._cache_full_process_area else: if zoom not in self.init_zoom_levels: - raise ValueError("zoom level not available with current configuration") + raise ValueError( + f"zoom level {zoom} not available with current configuration: {self.init_zoom_levels}" + ) return self._area_at_zoom(zoom) def _area_at_zoom(self, zoom): diff --git a/test/test_commands.py b/test/test_commands.py new file mode 100644 index 00000000..b8b5c069 --- /dev/null +++ b/test/test_commands.py @@ -0,0 +1,43 @@ +import os + +import mapchete +from mapchete.commands import cp + + +SCRIPTDIR = os.path.dirname(os.path.realpath(__file__)) +TESTDATA_DIR = os.path.join(SCRIPTDIR, "testdata") + + +def test_cp(mp_tmpdir, cleantopo_br, wkt_geom): + # generate TileDirectory + with mapchete.open( + cleantopo_br.path, bounds=[169.19251592399996, -90, 180, -80.18582802550002] + ) as mp: + mp.batch_process(zoom=5) + out_path = os.path.join(TESTDATA_DIR, cleantopo_br.dict["output"]["path"]) + + # copy tiles and subset by bounds + cp( + out_path, + os.path.join(mp_tmpdir, "all"), + zoom=5, + bounds=[169.19251592399996, -90, 180, -80.18582802550002], + ) + + # copy all tiles + cp( + out_path, + os.path.join(mp_tmpdir, "all"), + zoom=5, + ) + + # copy tiles and subset by area + cp(out_path, os.path.join(mp_tmpdir, "all"), zoom=5, area=wkt_geom) + + # copy local tiles without using threads + cp(out_path, os.path.join(mp_tmpdir, "all"), zoom=5, multi=1) + + +def test_cp_http(mp_tmpdir, http_tiledir): + # copy tiles and subset by bounds + cp(http_tiledir, os.path.join(mp_tmpdir, "http"), zoom=1, bounds=[3, 1, 4, 2]) From 2a7d41503a8ed4278eed1b97779cbf7139aa6fa7 Mon Sep 17 00:00:00 2001 From: Joachim Ungar Date: Fri, 16 Jul 2021 09:59:31 +0200 Subject: [PATCH 3/9] add rm to commands package --- mapchete/_processing.py | 16 ---- mapchete/cli/default/cp.py | 2 + mapchete/cli/default/rm.py | 104 ++++++----------------- mapchete/cli/utils.py | 5 +- mapchete/commands/__init__.py | 3 +- mapchete/commands/_cp.py | 6 +- mapchete/commands/_job.py | 2 +- mapchete/commands/_rm.py | 156 ++++++++++++++++++++++++++++++++++ setup.py | 2 +- test/test_commands.py | 38 +++++++-- 10 files changed, 226 insertions(+), 108 deletions(-) create mode 100644 mapchete/commands/_rm.py diff --git a/mapchete/_processing.py b/mapchete/_processing.py index cb05f2af..dd51d967 100644 --- a/mapchete/_processing.py +++ b/mapchete/_processing.py @@ -21,22 +21,6 @@ ProcessInfo = namedtuple("ProcessInfo", "tile processed process_msg written write_msg") -class ResultGenerator: - """ - Class to encapsulate processing generator while exposing generator length. - """ - - def __init__(self, generator, length): - self._generator = generator - self._length = length - - def __len__(self): - return self._length - - def __iter__(self): - return self._generator - - class TileProcess: """ Class to process on a specific process tile. diff --git a/mapchete/cli/default/cp.py b/mapchete/cli/default/cp.py index 15060344..b49cd516 100644 --- a/mapchete/cli/default/cp.py +++ b/mapchete/cli/default/cp.py @@ -25,6 +25,7 @@ @utils.opt_dst_fs_opts def cp(*args, debug=False, no_pbar=False, verbose=False, logfile=None, **kwargs): """Copy TileDirectory.""" + # handle deprecated options for x in ["password", "username"]: if kwargs.get(x): # pragma: no cover raise click.BadOptionUsage( @@ -32,6 +33,7 @@ def cp(*args, debug=False, no_pbar=False, verbose=False, logfile=None, **kwargs) f"'--{x} foo' is deprecated. You should use '--src-fs-opts {x}=foo' instead.", ) kwargs.pop(x) + # copy list( tqdm.tqdm( commands.cp( diff --git a/mapchete/cli/default/rm.py b/mapchete/cli/default/rm.py index 2a42d94b..9d3ef37f 100644 --- a/mapchete/cli/default/rm.py +++ b/mapchete/cli/default/rm.py @@ -1,22 +1,17 @@ import click -import logging import tqdm -import mapchete +from mapchete import commands from mapchete.cli import utils -from mapchete.io import fs_from_path, rm, tiles_exist -logger = logging.getLogger(__name__) - -@click.command("rm", help="Remove tiles from TileDirectory.") -@utils.arg_input +@click.command(help="Remove tiles from TileDirectory.") +@utils.arg_tiledir @utils.opt_zoom @utils.opt_area @utils.opt_area_crs @utils.opt_bounds @utils.opt_bounds_crs -@utils.opt_wkt_geometry @utils.opt_multi @utils.opt_verbose @utils.opt_no_pbar @@ -24,75 +19,32 @@ @utils.opt_logfile @utils.opt_force @utils.opt_fs_opts -def rm_( - input_, - zoom=None, - area=None, - area_crs=None, - bounds=None, - bounds_crs=None, - wkt_geometry=None, - multi=None, - verbose=False, - no_pbar=False, +def rm( + *args, + force=False, debug=False, + no_pbar=False, + verbose=False, logfile=None, - force=None, - fs_opts=None, + **kwargs, ): - """Copy TileDirectory.""" - if zoom is None: # pragma: no cover - raise click.UsageError("zoom level(s) required") - - src_fs = fs_from_path(input_, **fs_opts) - - # open source tile directory - with mapchete.open( - input_, - zoom=zoom, - area=area, - area_crs=area_crs, - bounds=bounds, - bounds_crs=bounds_crs, - wkt_geometry=wkt_geometry, - fs=src_fs, - fs_kwargs=fs_opts, - mode="readonly", - ) as src_mp: - tp = src_mp.config.output_pyramid - - tiles = {} - for z in range(min(zoom), max(zoom) + 1): - tiles[z] = [] - # check which source tiles exist - logger.debug(f"looking for existing source tiles in zoom {z}...") - for tile, exists in tiles_exist( - config=src_mp.config, - output_tiles=[ - t - for t in tp.tiles_from_geom(src_mp.config.area_at_zoom(z), z) - # this is required to omit tiles touching the config area - if src_mp.config.area_at_zoom(z).intersection(t.bbox).area - ], - multi=multi, - ): - if exists: - tiles[z].append(tile) - - total_tiles = sum([len(v) for v in tiles.values()]) - - if total_tiles: - if force or click.confirm( - f"Do you want to delete {total_tiles} tiles?", abort=True - ): - # remove - rm( - [ - src_mp.config.output_reader.get_path(tile) - for zoom_tiles in tiles.values() - for tile in zoom_tiles - ], - fs=src_fs, + """Remove tiles from TileDirectory.""" + tiles_to_delete = commands.rm( + *args, + as_iterator=True, + msg_callback=tqdm.tqdm.write if verbose else None, + **kwargs, + ) + if len(tiles_to_delete): + if force or click.confirm( + f"Do you want to delete {len(tiles_to_delete)} tiles?", abort=True + ): + list( + tqdm.tqdm( + tiles_to_delete, + unit="tile", + disable=debug or no_pbar, ) - else: # pragma: no cover - click.echo("No tiles found to delete.") + ) + else: + tqdm.tqdm.write("No tiles found to delete.") diff --git a/mapchete/cli/utils.py b/mapchete/cli/utils.py index 6d648f88..df091672 100644 --- a/mapchete/cli/utils.py +++ b/mapchete/cli/utils.py @@ -149,12 +149,11 @@ def _cb_key_val(ctx, param, value): arg_input_raster = click.argument("input_raster", type=click.Path(exists=True)) arg_out_dir = click.argument("output_dir", type=click.Path()) arg_input = click.argument("input_", metavar="INPUT", type=click.STRING) -arg_inputs = click.argument( - "inputs", metavar="INPUTS", nargs=-1, callback=_validate_inputs -) +arg_inputs = click.argument("inputs", nargs=-1, callback=_validate_inputs) arg_output = click.argument("output", type=click.STRING) arg_src_tiledir = click.argument("src_tiledir", type=click.STRING) arg_dst_tiledir = click.argument("dst_tiledir", type=click.STRING) +arg_tiledir = click.argument("tiledir", type=click.STRING) # click options # diff --git a/mapchete/commands/__init__.py b/mapchete/commands/__init__.py index ac394213..a05beb1b 100644 --- a/mapchete/commands/__init__.py +++ b/mapchete/commands/__init__.py @@ -3,6 +3,7 @@ This should make the use from within other scripts, notebooks, etc. easier. """ from mapchete.commands._cp import cp +from mapchete.commands._rm import rm -__all__ = ["cp"] +__all__ = ["cp", "rm"] diff --git a/mapchete/commands/_cp.py b/mapchete/commands/_cp.py index 51c8a59d..2c170e52 100644 --- a/mapchete/commands/_cp.py +++ b/mapchete/commands/_cp.py @@ -74,16 +74,16 @@ def cp( Examples -------- - >>> cp("foo", "bar") + >>> cp("foo", "bar", zoom=5) This will run the whole copy process. - >>> for i in cp("foo", "bar", as_iterator=True): + >>> for i in cp("foo", "bar", zoom=5, as_iterator=True): >>> print(i) This will return a generator where through iteration, tiles are copied. - >>> list(tqdm.tqdm(cp("foo", "bar", as_iterator=True))) + >>> list(tqdm.tqdm(cp("foo", "bar", zoom=5, as_iterator=True))) Usage within a process bar. """ diff --git a/mapchete/commands/_job.py b/mapchete/commands/_job.py index 0b33a4aa..21f8251a 100644 --- a/mapchete/commands/_job.py +++ b/mapchete/commands/_job.py @@ -24,7 +24,7 @@ def __len__(self): return self._total def __iter__(self): - if not self._as_iterator: + if not self._as_iterator: # pragma: no cover raise TypeError("initialize with 'as_iterator=True'") return self.func(*self.fargs, **self.fkwargs) diff --git a/mapchete/commands/_rm.py b/mapchete/commands/_rm.py new file mode 100644 index 00000000..132e2ab1 --- /dev/null +++ b/mapchete/commands/_rm.py @@ -0,0 +1,156 @@ +import logging +from rasterio.crs import CRS +from shapely.geometry.base import BaseGeometry +from typing import Callable, List, Tuple, Union + +import mapchete +from mapchete.cli import utils +from mapchete.commands._job import empty_callback, Job +from mapchete.io import fs_from_path, tiles_exist + +logger = logging.getLogger(__name__) + + +def rm( + tiledir: str, + zoom: Union[int, List[int]] = None, + area: Union[BaseGeometry, str, dict] = None, + area_crs: str = None, + bounds: Tuple[float] = None, + bounds_crs: Tuple[CRS, str] = None, + overwrite: bool = False, + multi: int = None, + fs_opts: dict = None, + msg_callback: Callable = None, + as_iterator: bool = False, +) -> Job: + """ + Remove tiles from TileDirectory. + + Parameters + ---------- + tiledir : str + TileDirectory or mapchete file. + zoom : integer or list of integers + Single zoom, minimum and maximum zoom or a list of zoom levels. + area : str, dict, BaseGeometry + Geometry to override bounds or area provided in process configuration. Can be either a + WKT string, a GeoJSON mapping, a shapely geometry or a path to a Fiona-readable file. + area_crs : CRS or str + CRS of area (default: process CRS). + bounds : tuple + Override bounds or area provided in process configuration. + bounds_crs : CRS or str + CRS of area (default: process CRS). + multi : int + Number of threads used to check whether tiles exist. + fs_opts : dict + Configuration options for fsspec filesystem. + msg_callback : Callable + Optional callback function for process messages. + as_iterator : bool + Returns as generator but with a __len__() property. + + Returns + ------- + Job instance either with already processed items or a generator with known length. + + Examples + -------- + >>> rm("foo", zoom=5) + + This will run the whole rm process. + + >>> for i in rm("foo", zoom=5, as_iterator=True): + >>> print(i) + + This will return a generator where through iteration, tiles are removed. + + >>> list(tqdm.tqdm(rm("foo", zoom=5, as_iterator=True))) + + Usage within a process bar. + + """ + msg_callback = msg_callback or empty_callback + fs_opts = fs_opts or {} + if zoom is None: # pragma: no cover + raise ValueError("zoom level(s) required") + + fs = fs_from_path(tiledir, **fs_opts) + + with mapchete.open( + tiledir, + zoom=zoom, + area=area, + area_crs=area_crs, + bounds=bounds, + bounds_crs=bounds_crs, + fs=fs, + fs_kwargs=fs_opts, + mode="readonly", + ) as mp: + tp = mp.config.output_pyramid + + tiles = {} + for z in mp.config.init_zoom_levels: + tiles[z] = [] + # check which source tiles exist + logger.debug(f"looking for existing source tiles in zoom {z}...") + for tile, exists in tiles_exist( + config=mp.config, + output_tiles=[ + t + for t in tp.tiles_from_geom(mp.config.area_at_zoom(z), z) + # this is required to omit tiles touching the config area + if mp.config.area_at_zoom(z).intersection(t.bbox).area + ], + multi=multi, + ): + if exists: + tiles[z].append(tile) + + paths = [ + mp.config.output_reader.get_path(tile) + for zoom_tiles in tiles.values() + for tile in zoom_tiles + ] + return Job( + _rm, + paths, + fs, + msg_callback, + as_iterator=as_iterator, + total=len(paths), + ) + + +def _rm(paths, fs, msg_callback, recursive=False): + """ + Remove one or multiple paths from file system. + + Note: all paths have to be from the same file system! + + Parameters + ---------- + paths : str or list + fs : fsspec.FileSystem + """ + logger.debug(f"got {len(paths)} path(s) on {fs}") + + # s3fs enables multiple paths as input, so let's use this: + if "s3" in fs.protocol: # pragma: no cover + fs.rm(paths, recursive=recursive) + for path in paths: + msg = f"deleted {path}" + logger.debug(msg) + yield msg + + # otherwise, just iterate through the paths + else: + for path in paths: + fs.rm(path, recursive=recursive) + msg = f"deleted {path}" + logger.debug(msg) + yield msg + + msg_callback(f"{len(paths)} tiles deleted") diff --git a/setup.py b/setup.py index 4d6288bf..71dadc38 100644 --- a/setup.py +++ b/setup.py @@ -66,7 +66,7 @@ "formats=mapchete.cli.default.formats:formats", "index=mapchete.cli.default.index:index", "processes=mapchete.cli.default.processes:processes", - "rm=mapchete.cli.default.rm:rm_", + "rm=mapchete.cli.default.rm:rm", "serve=mapchete.cli.default.serve:serve", ], "mapchete.formats.drivers": [ diff --git a/test/test_commands.py b/test/test_commands.py index b8b5c069..fe726564 100644 --- a/test/test_commands.py +++ b/test/test_commands.py @@ -1,7 +1,7 @@ import os import mapchete -from mapchete.commands import cp +from mapchete.commands import cp, rm SCRIPTDIR = os.path.dirname(os.path.realpath(__file__)) @@ -17,27 +17,51 @@ def test_cp(mp_tmpdir, cleantopo_br, wkt_geom): out_path = os.path.join(TESTDATA_DIR, cleantopo_br.dict["output"]["path"]) # copy tiles and subset by bounds - cp( + tiles = cp( out_path, - os.path.join(mp_tmpdir, "all"), + os.path.join(mp_tmpdir, "bounds"), zoom=5, bounds=[169.19251592399996, -90, 180, -80.18582802550002], ) + assert len(tiles) # copy all tiles - cp( + tiles = cp( out_path, os.path.join(mp_tmpdir, "all"), zoom=5, ) + assert len(tiles) # copy tiles and subset by area - cp(out_path, os.path.join(mp_tmpdir, "all"), zoom=5, area=wkt_geom) + tiles = cp(out_path, os.path.join(mp_tmpdir, "area"), zoom=5, area=wkt_geom) + assert len(tiles) # copy local tiles without using threads - cp(out_path, os.path.join(mp_tmpdir, "all"), zoom=5, multi=1) + tiles = cp(out_path, os.path.join(mp_tmpdir, "nothreads"), zoom=5, multi=1) + assert len(tiles) def test_cp_http(mp_tmpdir, http_tiledir): # copy tiles and subset by bounds - cp(http_tiledir, os.path.join(mp_tmpdir, "http"), zoom=1, bounds=[3, 1, 4, 2]) + tiles = cp( + http_tiledir, os.path.join(mp_tmpdir, "http"), zoom=1, bounds=[3, 1, 4, 2] + ) + assert len(tiles) + + +def test_rm(mp_tmpdir, cleantopo_br): + # generate TileDirectory + with mapchete.open( + cleantopo_br.path, bounds=[169.19251592399996, -90, 180, -80.18582802550002] + ) as mp: + mp.batch_process(zoom=5) + out_path = os.path.join(TESTDATA_DIR, cleantopo_br.dict["output"]["path"]) + + # remove tiles + tiles = rm(out_path, zoom=5) + assert len(tiles) > 0 + + # remove tiles but this time they should already have been removed + tiles = rm(out_path, zoom=5) + assert len(tiles) == 0 From c4348e587ff03a4ff76b68cd8ca80216a8620c2e Mon Sep 17 00:00:00 2001 From: Joachim Ungar Date: Fri, 16 Jul 2021 14:49:53 +0200 Subject: [PATCH 4/9] add index command --- mapchete/cli/default/index.py | 145 ++++++--------------------------- mapchete/commands/__init__.py | 3 +- mapchete/commands/_cp.py | 5 +- mapchete/commands/_index.py | 149 ++++++++++++++++++++++++++++++++++ mapchete/commands/_rm.py | 4 +- mapchete/index.py | 26 ++++-- mapchete/validate.py | 4 +- 7 files changed, 200 insertions(+), 136 deletions(-) create mode 100644 mapchete/commands/_index.py diff --git a/mapchete/cli/default/index.py b/mapchete/cli/default/index.py index 1babab07..ab6c15e6 100644 --- a/mapchete/cli/default/index.py +++ b/mapchete/cli/default/index.py @@ -9,7 +9,7 @@ import mapchete from mapchete.cli import utils -from mapchete.index import zoom_index_gen +from mapchete import commands # workaround for https://github.com/tqdm/tqdm/issues/481 @@ -19,7 +19,7 @@ @click.command(help="Create index of output tiles.") -@utils.arg_inputs +@utils.arg_tiledir @utils.opt_idx_out_dir @utils.opt_geojson @utils.opt_gpkg @@ -36,129 +36,34 @@ @utils.opt_area_crs @utils.opt_point @utils.opt_point_crs -@utils.opt_wkt_geometry @utils.opt_tile @utils.opt_http_username @utils.opt_http_password +@utils.opt_fs_opts @utils.opt_verbose @utils.opt_no_pbar @utils.opt_debug @utils.opt_logfile -def index( - inputs, - idx_out_dir=None, - geojson=False, - gpkg=False, - shp=False, - vrt=False, - txt=False, - fieldname=None, - basepath=None, - for_gdal=False, - zoom=None, - bounds=None, - bounds_crs=None, - area=None, - area_crs=None, - point=None, - point_crs=None, - wkt_geometry=None, - tile=None, - username=None, - password=None, - verbose=False, - no_pbar=False, - debug=False, - logfile=None, -): - if not any([geojson, gpkg, shp, txt, vrt]): - raise click.MissingParameter( - """At least one of '--geojson', '--gpkg', '--shp', '--vrt' or '--txt'""" - """must be provided.""", - param_type="option", +def index(*args, debug=False, no_pbar=False, verbose=False, logfile=None, **kwargs): + """Create various index files from process output.""" + # handle deprecated options + for x in ["password", "username"]: + if kwargs.get(x): # pragma: no cover + raise click.BadOptionUsage( + x, + f"'--{x} foo' is deprecated. You should use '--fs-opts {x}=foo' instead.", + ) + kwargs.pop(x) + list( + tqdm.tqdm( + commands.index( + *args, + as_iterator=True, + msg_callback=tqdm.tqdm.write if verbose else None, + **kwargs, + ), + unit="tile", + disable=debug or no_pbar, ) - # send verbose messages to /dev/null if not activated - verbose_dst = open(os.devnull, "w") if debug or not verbose else sys.stdout - - for input_ in inputs: - - tqdm.tqdm.write("create index(es) for %s" % input_, file=verbose_dst) - - with click_spinner.spinner(disable=debug) as spinner: - - # process single tile - if tile: - with mapchete.open( - input_, mode="readonly", username=username, password=password - ) as mp: - spinner.stop() - tile = mp.config.process_pyramid.tile(*tile) - for tile in tqdm.tqdm( - zoom_index_gen( - mp=mp, - zoom=tile.zoom, - out_dir=idx_out_dir - if idx_out_dir - else mp.config.output.path, - geojson=geojson, - gpkg=gpkg, - shapefile=shp, - vrt=vrt, - txt=txt, - fieldname=fieldname, - basepath=basepath, - for_gdal=for_gdal, - ), - total=mp.count_tiles(tile.zoom, tile.zoom), - unit="tile", - disable=debug or no_pbar, - ): - logger.debug("%s indexed", tile) - - # process area - else: - with mapchete.open( - input_, - mode="readonly", - zoom=zoom, - wkt_geometry=wkt_geometry, - point=point, - point_crs=point_crs, - bounds=bounds, - bounds_crs=bounds_crs, - area=area, - area_crs=area_crs, - username=username, - password=password, - ) as mp: - spinner.stop() - logger.debug("process bounds: %s", mp.config.init_bounds) - logger.debug("process area: %s", mp.config.init_area) - logger.debug("process zooms: %s", mp.config.init_zoom_levels) - logger.debug("fieldname: %s", fieldname) - for tile in tqdm.tqdm( - zoom_index_gen( - mp=mp, - zoom=mp.config.init_zoom_levels, - out_dir=( - idx_out_dir if idx_out_dir else mp.config.output.path - ), - geojson=geojson, - gpkg=gpkg, - shapefile=shp, - vrt=vrt, - txt=txt, - fieldname=fieldname, - basepath=basepath, - for_gdal=for_gdal, - ), - total=mp.count_tiles( - min(mp.config.init_zoom_levels), - max(mp.config.init_zoom_levels), - ), - unit="tile", - disable=debug or no_pbar, - ): - logger.debug("%s indexed", tile) - - tqdm.tqdm.write("index(es) creation for %s finished" % input_, file=verbose_dst) + ) + tqdm.tqdm.write(f"index(es) creation for {kwargs.get('tiledir')} finished") diff --git a/mapchete/commands/__init__.py b/mapchete/commands/__init__.py index a05beb1b..16f5d5e7 100644 --- a/mapchete/commands/__init__.py +++ b/mapchete/commands/__init__.py @@ -3,7 +3,8 @@ This should make the use from within other scripts, notebooks, etc. easier. """ from mapchete.commands._cp import cp +from mapchete.commands._index import index from mapchete.commands._rm import rm -__all__ = ["cp", "rm"] +__all__ = ["cp", "index", "rm"] diff --git a/mapchete/commands/_cp.py b/mapchete/commands/_cp.py index 2c170e52..bc192076 100644 --- a/mapchete/commands/_cp.py +++ b/mapchete/commands/_cp.py @@ -7,7 +7,6 @@ from shapely.geometry.base import BaseGeometry from shapely.ops import unary_union from tilematrix import TilePyramid -import tqdm from typing import Callable, List, Tuple, Union import mapchete @@ -25,9 +24,9 @@ def cp( dst_tiledir: str, zoom: Union[int, List[int]] = None, area: Union[BaseGeometry, str, dict] = None, - area_crs: str = None, + area_crs: Union[CRS, str] = None, bounds: Tuple[float] = None, - bounds_crs: Tuple[CRS, str] = None, + bounds_crs: Union[CRS, str] = None, overwrite: bool = False, multi: int = None, src_fs_opts: dict = None, diff --git a/mapchete/commands/_index.py b/mapchete/commands/_index.py new file mode 100644 index 00000000..18441dd3 --- /dev/null +++ b/mapchete/commands/_index.py @@ -0,0 +1,149 @@ +"""Create index for process output.""" + +import logging +import os +from rasterio.crs import CRS +from shapely.geometry.base import BaseGeometry +import sys +import tqdm +from typing import Callable, List, Tuple, Union + +import mapchete +from mapchete.commands._job import empty_callback, Job +from mapchete.cli import utils +from mapchete.index import zoom_index_gen + +logger = logging.getLogger(__name__) + + +def index( + tiledir: str, + idx_out_dir: str = None, + geojson: bool = False, + gpkg: bool = False, + shp: bool = False, + vrt: bool = False, + txt: bool = False, + fieldname: str = None, + basepath: str = None, + for_gdal: bool = False, + zoom: Union[int, List[int]] = None, + area: Union[BaseGeometry, str, dict] = None, + area_crs: Union[CRS, str] = None, + bounds: Tuple[float] = None, + bounds_crs: Union[CRS, str] = None, + point: Tuple[float, float] = None, + point_crs: Tuple[float, float] = None, + tile: Tuple[int, int, int] = None, + fs_opts: dict = None, + msg_callback: Callable = None, + as_iterator: bool = False, +) -> Job: + """ + Create one or more indexes from a TileDirectory. + + Parameters + ---------- + tiledir : str + Source TileDirectory or mapchete file. + idx_out_dir : str + Alternative output dir for index. Defaults to TileDirectory path. + geojson : bool + Activate GeoJSON output. + gpkg : bool + Activate GeoPackage output. + shp : bool + Activate Shapefile output. + vrt : bool + Activate VRT output. + txt : bool + Activate TXT output. + fieldname : str + Field name which contains paths of tiles (default: "location"). + basepath : str + Use custom base path for absolute paths instead of output path. + for_gdal : bool + Use GDAL compatible remote paths, i.e. add "/vsicurl/" before path. + zoom : integer or list of integers + Single zoom, minimum and maximum zoom or a list of zoom levels. + area : str, dict, BaseGeometry + Geometry to override bounds or area provided in process configuration. Can be either a + WKT string, a GeoJSON mapping, a shapely geometry or a path to a Fiona-readable file. + area_crs : CRS or str + CRS of area (default: process CRS). + bounds : tuple + Override bounds or area provided in process configuration. + bounds_crs : CRS or str + CRS of area (default: process CRS). + point : iterable + X and y coordinates of point whose corresponding process tile bounds will be used. + point_crs : str or CRS + CRS of point (defaults to process pyramid CRS). + tile : tuple + Zoom, row and column of tile to be processed (cannot be used with zoom) + fs_opts : dict + Configuration options for fsspec filesystem. + msg_callback : Callable + Optional callback function for process messages. + as_iterator : bool + Returns as generator but with a __len__() property. + + Returns + ------- + Job instance either with already processed items or a generator with known length. + + Examples + -------- + >>> index("foo", vrt=True, zoom=5) + + This will run the whole copy process. + + >>> for i in index("foo", vrt=True, zoom=5, as_iterator=True): + >>> print(i) + + This will return a generator where through iteration, tiles are copied. + + >>> list(tqdm.tqdm(index("foo", vrt=True, zoom=5, as_iterator=True))) + + Usage within a process bar. + """ + + if not any([geojson, gpkg, shp, txt, vrt]): + raise ValueError( + """At least one of '--geojson', '--gpkg', '--shp', '--vrt' or '--txt'""" + """must be provided.""" + ) + msg_callback = msg_callback or empty_callback + fs_opts = fs_opts or {} + + msg_callback(f"create index(es) for {tiledir}") + # process single tile + with mapchete.open( + tiledir, + mode="readonly", + fs_kwargs=fs_opts, + zoom=tile[0] if tile else zoom, + point=point, + point_crs=point_crs, + bounds=bounds, + bounds_crs=bounds_crs, + area=area, + area_crs=area_crs, + ) as mp: + return Job( + zoom_index_gen, + mp=mp, + zoom=mp.config.init_zoom_levels, + tile=tile, + out_dir=idx_out_dir if idx_out_dir else mp.config.output.path, + geojson=geojson, + gpkg=gpkg, + shapefile=shp, + vrt=vrt, + txt=txt, + fieldname=fieldname, + basepath=basepath, + for_gdal=for_gdal, + as_iterator=as_iterator, + total=1 if tile else mp.count_tiles(), + ) diff --git a/mapchete/commands/_rm.py b/mapchete/commands/_rm.py index 132e2ab1..b9213923 100644 --- a/mapchete/commands/_rm.py +++ b/mapchete/commands/_rm.py @@ -15,9 +15,9 @@ def rm( tiledir: str, zoom: Union[int, List[int]] = None, area: Union[BaseGeometry, str, dict] = None, - area_crs: str = None, + area_crs: Union[CRS, str] = None, bounds: Tuple[float] = None, - bounds_crs: Tuple[CRS, str] = None, + bounds_crs: Union[CRS, str] = None, overwrite: bool = False, multi: int = None, fs_opts: dict = None, diff --git a/mapchete/index.py b/mapchete/index.py index 4fdf4c10..f4e88465 100644 --- a/mapchete/index.py +++ b/mapchete/index.py @@ -52,6 +52,7 @@ def zoom_index_gen( mp=None, out_dir=None, zoom=None, + tile=None, geojson=False, gpkg=False, shapefile=False, @@ -149,16 +150,23 @@ def zoom_index_gen( # all output tiles for given process area logger.debug("determine affected output tiles") - output_tiles = set( - [ - tile - for tile in mp.config.output_pyramid.tiles_from_geom( - mp.config.area_at_zoom(zoom), zoom + if tile: + output_tiles = set( + mp.config.output_pyramid.intersecting( + mp.config.process_pyramid.tile(*tile) ) - # this is required to omit tiles touching the config area - if tile.bbox.intersection(mp.config.area_at_zoom(zoom)).area - ] - ) + ) + else: + output_tiles = set( + [ + tile + for tile in mp.config.output_pyramid.tiles_from_geom( + mp.config.area_at_zoom(zoom), zoom + ) + # this is required to omit tiles touching the config area + if tile.bbox.intersection(mp.config.area_at_zoom(zoom)).area + ] + ) # check which tiles exist in any index logger.debug("check which tiles exist in index(es)") diff --git a/mapchete/validate.py b/mapchete/validate.py index ce4289b2..81b9971c 100644 --- a/mapchete/validate.py +++ b/mapchete/validate.py @@ -124,7 +124,9 @@ def validate_values(config, values): if value not in config: raise ValueError("%s not given" % value) if not isinstance(config[value], vtype): - raise TypeError("%s must be %s" % (value, vtype)) + if config[value] is None: + raise ValueError("%s not given" % value) + raise TypeError("%s must be %s, not %s" % (value, vtype, config[value])) return True From 274d55d6954eeda562340e934d27cf428affb7d3 Mon Sep 17 00:00:00 2001 From: Joachim Ungar Date: Fri, 16 Jul 2021 18:51:01 +0200 Subject: [PATCH 5/9] add execute and convert --- mapchete/_core.py | 4 +- mapchete/cli/default/convert.py | 339 ++++----------------------- mapchete/cli/default/execute.py | 120 ++++------ mapchete/cli/utils.py | 170 -------------- mapchete/commands/__init__.py | 4 +- mapchete/commands/_convert.py | 400 ++++++++++++++++++++++++++++++++ mapchete/commands/_execute.py | 155 +++++++++++++ mapchete/commands/_index.py | 3 +- test/test_cli.py | 71 ++---- 9 files changed, 671 insertions(+), 595 deletions(-) mode change 100755 => 100644 mapchete/cli/default/execute.py create mode 100644 mapchete/commands/_convert.py create mode 100755 mapchete/commands/_execute.py diff --git a/mapchete/_core.py b/mapchete/_core.py index ed7aef5c..7b30d1ff 100644 --- a/mapchete/_core.py +++ b/mapchete/_core.py @@ -76,7 +76,7 @@ def open(some_input, with_cache=False, fs=None, fs_kwargs=None, **kwargs): path=some_input, fs=fs, fs_kwargs=fs_kwargs, - **kwargs + **kwargs, ), config_dir=os.getcwd(), zoom_levels=kwargs.get("zoom"), @@ -545,8 +545,10 @@ def __exit__(self, exc_type, exc_value, exc_traceback): # run input drivers cleanup for ip in self.config.input.values(): if ip is not None: + logger.debug(f"running cleanup on {ip}...") ip.cleanup() # run output driver cleanup + logger.debug(f"closing output driver {self.config.output}...") self.config.output.close( exc_type=exc_type, exc_value=exc_value, exc_traceback=exc_traceback ) diff --git a/mapchete/cli/default/convert.py b/mapchete/cli/default/convert.py index a550884b..393dcce6 100644 --- a/mapchete/cli/default/convert.py +++ b/mapchete/cli/default/convert.py @@ -1,30 +1,15 @@ import click -import fiona -import logging -from multiprocessing import cpu_count -import os -from pprint import pformat -import rasterio -from rasterio.enums import Resampling from rasterio.dtypes import dtype_ranges +from rasterio.enums import Resampling from rasterio.rio.options import creation_options -from shapely.geometry import box -import sys import tilematrix +import tqdm +import mapchete +from mapchete import commands from mapchete.cli import utils -from mapchete.config import raw_conf, raw_conf_output_pyramid -from mapchete.formats import ( - driver_from_file, - available_output_formats, - available_input_formats, -) -from mapchete.io import read_json, get_best_zoom_level -from mapchete.io.vector import reproject_geometry -from mapchete.tile import BufferedTilePyramid -from mapchete.validate import validate_zooms +from mapchete.formats import available_output_formats -logger = logging.getLogger(__name__) OUTPUT_FORMATS = available_output_formats() @@ -37,7 +22,7 @@ def _validate_bidx(ctx, param, bidx): @click.command(help="Convert outputs or other geodata.") -@utils.arg_input +@utils.arg_tiledir @utils.arg_output @utils.opt_zoom @utils.opt_bounds @@ -46,12 +31,11 @@ def _validate_bidx(ctx, param, bidx): @utils.opt_area_crs @utils.opt_point @utils.opt_point_crs -@utils.opt_wkt_geometry @click.option( "--clip-geometry", "-c", type=click.Path(exists=True), - help="Clip output by geometry", + help="Clip output by geometry.", ) @click.option("--bidx", callback=_validate_bidx, help="Band indexes to copy.") @click.option( @@ -66,7 +50,7 @@ def _validate_bidx(ctx, param, bidx): ) @click.option( "--output-format", - type=click.Choice(available_output_formats()), + type=click.Choice(OUTPUT_FORMATS), help="Output format.", ) @click.option( @@ -116,277 +100,52 @@ def _validate_bidx(ctx, param, bidx): @utils.opt_vrt @utils.opt_idx_out_dir def convert( - input_, + tiledir, output, - zoom=None, - bounds=None, - bounds_crs=None, - area=None, - area_crs=None, - point=None, - point_crs=None, - wkt_geometry=None, - clip_geometry=None, - bidx=None, - output_pyramid=None, - output_metatiling=None, - output_format=None, - output_dtype=None, - output_geometry_type=None, - creation_options=None, - scale_ratio=None, - scale_offset=None, - resampling_method=None, - overviews=False, - overviews_resampling_method=None, - cog=False, - overwrite=False, - logfile=None, - verbose=False, - no_pbar=False, - debug=False, - multi=None, + *args, vrt=False, idx_out_dir=None, + debug=False, + no_pbar=False, + verbose=False, + logfile=None, + **kwargs, ): - try: - input_info = _get_input_info(input_) - output_info = _get_output_info(output) - except Exception as e: - raise click.BadArgumentUsage(e) - - # collect mapchete configuration - mapchete_config = dict( - process="mapchete.processes.convert", - input=dict(inp=input_, clip=clip_geometry), - pyramid=( - dict( - grid=output_pyramid, - metatiling=( - output_metatiling - or ( - input_info["pyramid"].get("metatiling", 1) - if input_info["pyramid"] - else 1 - ) - ), - pixelbuffer=( - input_info["pyramid"].get("pixelbuffer", 0) - if input_info["pyramid"] - else 0 - ), - ) - if output_pyramid - else input_info["pyramid"] - ), - output=dict( - { - k: v - for k, v in input_info["output_params"].items() - if k not in ["delimiters", "bounds", "mode"] - }, - path=output, - format=( - output_format - or output_info["driver"] - or input_info["output_params"]["format"] - ), - dtype=output_dtype or input_info["output_params"].get("dtype"), - **creation_options, - **dict(overviews=True, overviews_resampling=overviews_resampling_method) - if overviews - else dict(), - ), - config_dir=os.getcwd(), - zoom_levels=zoom or input_info["zoom_levels"], - scale_ratio=scale_ratio, - scale_offset=scale_offset, - resampling=resampling_method, - band_indexes=bidx, - ) - - # assert all required information is there - if mapchete_config["output"]["format"] is None: - # this happens if input file is e.g. JPEG2000 and output is a tile directory - raise click.BadOptionUsage("output-format", "Output format required.") - if mapchete_config["output"]["format"] == "GTiff": - mapchete_config["output"].update(cog=cog) - output_type = OUTPUT_FORMATS[mapchete_config["output"]["format"]]["data_type"] - if bidx is not None: - mapchete_config["output"].update(bands=len(bidx)) - if mapchete_config["pyramid"] is None: - raise click.BadOptionUsage("output-pyramid", "Output pyramid required.") - elif mapchete_config["zoom_levels"] is None: - try: - mapchete_config.update( - zoom_levels=dict( - min=0, - max=get_best_zoom_level(input_, mapchete_config["pyramid"]["grid"]), - ) - ) - except Exception: - raise click.BadOptionUsage("zoom", "Zoom levels required.") - elif input_info["input_type"] != output_type: - raise click.BadArgumentUsage( - "Output format type (%s) is incompatible with input format (%s)." - % (output_type, input_info["input_type"]) + with mapchete.Timer() as t: + job = commands.convert( + tiledir, + output, + *args, + as_iterator=True, + msg_callback=tqdm.tqdm.write if verbose else None, + **kwargs, ) - if output_metatiling: - mapchete_config["output"].update(metatiling=output_metatiling) - if input_info["output_params"].get("schema") and output_geometry_type: - mapchete_config["output"]["schema"].update(geometry=output_geometry_type) - - # determine process bounds - out_pyramid = BufferedTilePyramid.from_dict(mapchete_config["pyramid"]) - inp_bounds = ( - bounds - or reproject_geometry( - box(*input_info["bounds"]), - src_crs=input_info["crs"], - dst_crs=out_pyramid.crs, - ).bounds - if input_info["bounds"] - else out_pyramid.bounds - ) - # if clip-geometry is available, intersect determined bounds with clip bounds - if clip_geometry: - clip_intersection = _clip_bbox( - clip_geometry, dst_crs=out_pyramid.crs - ).intersection(box(*inp_bounds)) - if clip_intersection.is_empty: - click.echo( - "Process area is empty: clip bounds don't intersect with input bounds." - ) + if not len(job): return - # add process bounds and output type - mapchete_config.update( - bounds=(clip_intersection.bounds if clip_geometry else inp_bounds), - bounds_crs=bounds_crs, - clip_to_output_dtype=mapchete_config["output"].get("dtype", None), - ) - logger.debug("temporary config generated: %s", pformat(mapchete_config)) - - utils._process_area( - debug=debug, - mapchete_config=mapchete_config, - mode="overwrite" if overwrite else "continue", - zoom=zoom, - wkt_geometry=wkt_geometry, - point=point, - point_crs=point_crs, - bounds=bounds, - bounds_crs=bounds_crs, - area=area, - area_crs=area_crs, - multi=multi or cpu_count(), - verbose_dst=open(os.devnull, "w") if debug or not verbose else sys.stdout, - no_pbar=no_pbar, - vrt=vrt, - idx_out_dir=idx_out_dir, - ) - - -def _clip_bbox(clip_geometry, dst_crs=None): - with fiona.open(clip_geometry) as src: - return reproject_geometry(box(*src.bounds), src_crs=src.crs, dst_crs=dst_crs) - - -def _get_input_info(input_): - - # assuming single file if path has a file extension - if os.path.splitext(input_)[1]: - driver = driver_from_file(input_) - - # single file input can be a mapchete file or a rasterio/fiona file - if driver == "Mapchete": - logger.debug("input is mapchete file") - input_info = _input_mapchete_info(input_) - - elif driver == "raster_file": - # this should be readable by rasterio - logger.debug("input is raster_file") - input_info = _input_rasterio_info(input_) - - elif driver == "vector_file": - # this should be readable by Fiona - input_info = _input_fiona_info(input_) - else: # pragma: no cover - raise NotImplementedError("driver %s is not supported" % driver) - - # assuming tile directory - else: - logger.debug("input is tile directory") - input_info = _input_tile_directory_info(input_) - - return input_info - - -def _input_mapchete_info(input_): - conf = raw_conf(input_) - output_params = conf["output"] - pyramid = raw_conf_output_pyramid(conf) - return dict( - output_params=output_params, - pyramid=pyramid.to_dict(), - crs=pyramid.crs, - zoom_levels=validate_zooms(conf["zoom_levels"], expand=False), - pixel_size=None, - input_type=OUTPUT_FORMATS[output_params["format"]]["data_type"], - bounds=conf.get("bounds"), - ) - - -def _input_rasterio_info(input_): - with rasterio.open(input_) as src: - return dict( - output_params=dict( - bands=src.meta["count"], - dtype=src.meta["dtype"], - format=src.driver if src.driver in available_input_formats() else None, - ), - pyramid=None, - crs=src.crs, - zoom_levels=None, - pixel_size=src.transform[0], - input_type="raster", - bounds=src.bounds, + list( + tqdm.tqdm( + job, + unit="tile", + disable=debug or no_pbar, + ) ) - - -def _input_fiona_info(input_): - with fiona.open(input_) as src: - return dict( - output_params=dict( - schema=src.schema, - format=src.driver if src.driver in available_input_formats() else None, - ), - pyramid=None, - crs=src.crs, - zoom_levels=None, - input_type="vector", - bounds=src.bounds, + tqdm.tqdm.write(f"processing {tiledir} finished in {t}") + + if vrt: + tqdm.tqdm.write("creating VRT(s)") + list( + tqdm.tqdm( + commands.index( + output, + *args, + vrt=vrt, + idx_out_dir=idx_out_dir, + as_iterator=True, + msg_callback=tqdm.tqdm.write if verbose else None, + **kwargs, + ), + unit="tile", + disable=debug or no_pbar, + ) ) - - -def _input_tile_directory_info(input_): - conf = read_json(os.path.join(input_, "metadata.json")) - pyramid = BufferedTilePyramid.from_dict(conf["pyramid"]) - return dict( - output_params=conf["driver"], - pyramid=pyramid.to_dict(), - crs=pyramid.crs, - zoom_levels=None, - pixel_size=None, - input_type=OUTPUT_FORMATS[conf["driver"]["format"]]["data_type"], - bounds=None, - ) - - -def _get_output_info(output): - _, file_ext = os.path.splitext(output) - if not file_ext: - return dict(type="TileDirectory", driver=None) - elif file_ext == ".tif": - return dict(type="SingleFile", driver="GTiff") - else: - raise TypeError("Could not determine output from extension: %s", file_ext) + tqdm.tqdm.write(f"index(es) creation for {tiledir} finished") diff --git a/mapchete/cli/default/execute.py b/mapchete/cli/default/execute.py old mode 100755 new mode 100644 index a863c664..d8455da5 --- a/mapchete/cli/default/execute.py +++ b/mapchete/cli/default/execute.py @@ -1,20 +1,9 @@ -"""Command line utility to execute a Mapchete process.""" - import click -import logging -from multiprocessing import cpu_count -import os -import sys import tqdm +import mapchete +from mapchete import commands from mapchete.cli import utils -from mapchete.config import raw_conf_process_pyramid - - -# workaround for https://github.com/tqdm/tqdm/issues/481 -tqdm.monitor_interval = 0 - -logger = logging.getLogger(__name__) @click.command(help="Execute a process.") @@ -26,7 +15,6 @@ @utils.opt_area_crs @utils.opt_point @utils.opt_point_crs -@utils.opt_wkt_geometry @utils.opt_tile @utils.opt_overwrite @utils.opt_multi @@ -41,68 +29,54 @@ @utils.opt_idx_out_dir def execute( mapchete_files, - zoom=None, - bounds=None, - bounds_crs=None, - area=None, - area_crs=None, - point=None, - point_crs=None, - wkt_geometry=None, - tile=None, - overwrite=False, - multi=None, - input_file=None, - logfile=None, - verbose=False, - no_pbar=False, - debug=False, - max_chunksize=None, - multiprocessing_start_method=None, + *args, vrt=False, idx_out_dir=None, + debug=False, + no_pbar=False, + verbose=False, + logfile=None, + input_file=None, + **kwargs, ): - """Execute a Mapchete process.""" - mode = "overwrite" if overwrite else "continue" - # send verbose messages to /dev/null if not activated - verbose_dst = open(os.devnull, "w") if debug or not verbose else sys.stdout - + if input_file is not None: # pragma: no cover + raise click.BadOptionUsage( + "input-file", + "'--input-file' is deprecated.", + ) for mapchete_file in mapchete_files: - tqdm.tqdm.write("preparing to process %s" % mapchete_file, file=verbose_dst) - # process single tile - if tile: - utils._process_single_tile( - raw_conf_process_pyramid=raw_conf_process_pyramid, - mapchete_config=mapchete_file, - tile=tile, - mode=mode, - input_file=input_file, - debug=debug, - verbose_dst=verbose_dst, - vrt=vrt, - idx_out_dir=idx_out_dir, - no_pbar=no_pbar, + tqdm.tqdm.write(f"preparing to process {mapchete_file}") + with mapchete.Timer() as t: + list( + tqdm.tqdm( + commands.execute( + mapchete_file, + *args, + as_iterator=True, + msg_callback=tqdm.tqdm.write if verbose else None, + **kwargs, + ), + unit="tile", + disable=debug or no_pbar, + ) ) - # process area - else: - utils._process_area( - debug=debug, - mapchete_config=mapchete_file, - mode=mode, - zoom=zoom, - wkt_geometry=wkt_geometry, - point=point, - point_crs=point_crs, - bounds=bounds, - bounds_crs=bounds_crs, - area=area, - area_crs=area_crs, - input_file=input_file, - multi=multi or cpu_count(), - verbose_dst=verbose_dst, - max_chunksize=max_chunksize, - multiprocessing_start_method=multiprocessing_start_method, - no_pbar=no_pbar, - vrt=vrt, - idx_out_dir=idx_out_dir, + tqdm.tqdm.write(f"processing {mapchete_file} finished in {t}") + + if vrt: + tqdm.tqdm.write("creating VRT(s)") + list( + tqdm.tqdm( + commands.index( + mapchete_file, + *args, + vrt=vrt, + idx_out_dir=idx_out_dir, + as_iterator=True, + msg_callback=tqdm.tqdm.write if verbose else None, + **kwargs, + ), + unit="tile", + disable=debug or no_pbar, + ) ) + tqdm.tqdm.write(f"index(es) creation for {mapchete_file} finished") diff --git a/mapchete/cli/utils.py b/mapchete/cli/utils.py index df091672..97b50470 100644 --- a/mapchete/cli/utils.py +++ b/mapchete/cli/utils.py @@ -373,173 +373,3 @@ def _cb_key_val(ctx, param, value): callback=_cb_key_val, help="Configuration options for destination fsspec filesystem.", ) - - -# convenience processing functions # -#################################### -def _process_single_tile( - debug=None, - raw_conf_process_pyramid=None, - mapchete_config=None, - tile=None, - mode=None, - input_file=None, - verbose_dst=None, - vrt=None, - idx_out_dir=None, - no_pbar=None, -): - with click_spinner.spinner(disable=debug) as spinner: - with mapchete.Timer() as t: - tile = raw_conf_process_pyramid(raw_conf(mapchete_config)).tile(*tile) - with mapchete.open( - mapchete_config, - mode=mode, - bounds=tile.bounds, - zoom=tile.zoom, - single_input_file=input_file, - ) as mp: - spinner.stop() - tqdm.tqdm.write("processing 1 tile", file=verbose_dst) - - # run process on tile - for result in mp.batch_processor(tile=tile): - write_verbose_msg(result, dst=verbose_dst) - - tqdm.tqdm.write( - ( - "processing %s finished in %s" % (mapchete_config, t) - if isinstance(mapchete_config, str) - else "processing finished in %s" % t - ), - file=verbose_dst, - ) - - # write VRT index - if vrt: - with mapchete.Timer() as t_vrt: - tqdm.tqdm.write("creating VRT", file=verbose_dst) - for tile in tqdm.tqdm( - zoom_index_gen( - mp=mp, - zoom=tile.zoom, - out_dir=idx_out_dir or mp.config.output.path, - vrt=vrt, - ), - total=mp.count_tiles(tile.zoom, tile.zoom), - unit="tile", - disable=debug or no_pbar, - ): - logger.debug("%s indexed", tile) - - tqdm.tqdm.write( - ( - "VRT(s) for %s created in %s" % (mapchete_config, t_vrt) - if isinstance(mapchete_config, str) - else "VRT(s) created in %s" % t_vrt - ), - file=verbose_dst, - ) - - -def _process_area( - debug=None, - mapchete_config=None, - mode=None, - zoom=None, - wkt_geometry=None, - point=None, - point_crs=None, - bounds=None, - bounds_crs=None, - area=None, - area_crs=None, - input_file=None, - multi=None, - verbose_dst=None, - max_chunksize=None, - multiprocessing_start_method=None, - no_pbar=None, - vrt=None, - idx_out_dir=None, -): - multi = multi or cpu_count() - with click_spinner.spinner(disable=debug) as spinner: - with mapchete.Timer() as t: - with mapchete.open( - mapchete_config, - mode=mode, - zoom=zoom, - bounds=bounds_from_opts( - wkt_geometry=wkt_geometry, - point=point, - point_crs=point_crs, - bounds=bounds, - bounds_crs=bounds_crs, - raw_conf=raw_conf(mapchete_config), - ), - area=area, - area_crs=area_crs, - single_input_file=input_file, - ) as mp: - spinner.stop() - tiles_count = mp.count_tiles( - min(mp.config.init_zoom_levels), max(mp.config.init_zoom_levels) - ) - - tqdm.tqdm.write( - "processing %s tile(s) on %s worker(s)" % (tiles_count, multi), - file=verbose_dst, - ) - - # run process on tiles - for process_info in tqdm.tqdm( - mp.batch_processor( - multi=multi, - zoom=zoom, - max_chunksize=max_chunksize, - multiprocessing_start_method=multiprocessing_start_method, - ), - total=tiles_count, - unit="tile", - disable=debug or no_pbar, - ): - write_verbose_msg(process_info, dst=verbose_dst) - - tqdm.tqdm.write( - ( - "processing %s finished in %s" % (mapchete_config, t) - if isinstance(mapchete_config, str) - else "processing finished in %s" % t - ), - file=verbose_dst, - ) - - # write VRT index - if vrt: - with mapchete.Timer() as t_vrt: - tqdm.tqdm.write("creating VRT(s)", file=verbose_dst) - for tile in tqdm.tqdm( - zoom_index_gen( - mp=mp, - zoom=mp.config.init_zoom_levels, - out_dir=idx_out_dir or mp.config.output.path, - vrt=vrt, - ), - total=mp.count_tiles( - min(mp.config.init_zoom_levels), - max(mp.config.init_zoom_levels), - ), - unit="tile", - disable=debug or no_pbar, - ): - logger.debug("%s indexed", tile) - - tqdm.tqdm.write( - ( - "VRT(s) for %s created in %s" % (mapchete_config, t_vrt) - if isinstance(mapchete_config, str) - else "VRT(s) created in %s" % t_vrt - ), - file=verbose_dst, - ) diff --git a/mapchete/commands/__init__.py b/mapchete/commands/__init__.py index 16f5d5e7..6a3d5012 100644 --- a/mapchete/commands/__init__.py +++ b/mapchete/commands/__init__.py @@ -2,9 +2,11 @@ This package contains easy to access functions which otherwise would have to be called via the CLI. This should make the use from within other scripts, notebooks, etc. easier. """ +from mapchete.commands._convert import convert from mapchete.commands._cp import cp +from mapchete.commands._execute import execute from mapchete.commands._index import index from mapchete.commands._rm import rm -__all__ = ["cp", "index", "rm"] +__all__ = ["convert", "cp", "execute", "index", "rm"] diff --git a/mapchete/commands/_convert.py b/mapchete/commands/_convert.py new file mode 100644 index 00000000..0bdc7433 --- /dev/null +++ b/mapchete/commands/_convert.py @@ -0,0 +1,400 @@ +import fiona +import logging +from multiprocessing import cpu_count +import os +from pprint import pformat +import rasterio +from rasterio.crs import CRS +from shapely.geometry import box +from shapely.geometry.base import BaseGeometry +import tilematrix +from typing import Callable, List, Tuple, Union + +import mapchete +from mapchete.commands._execute import execute +from mapchete.commands._job import empty_callback, Job +from mapchete.config import raw_conf, raw_conf_output_pyramid +from mapchete.formats import ( + driver_from_file, + available_output_formats, + available_input_formats, +) +from mapchete.io import read_json, get_best_zoom_level +from mapchete.io.vector import reproject_geometry +from mapchete.tile import BufferedTilePyramid +from mapchete.validate import validate_zooms + +logger = logging.getLogger(__name__) +OUTPUT_FORMATS = available_output_formats() + + +def convert( + tiledir: str, + output: str, + zoom: Union[int, List[int]] = None, + area: Union[BaseGeometry, str, dict] = None, + area_crs: Union[CRS, str] = None, + bounds: Tuple[float] = None, + bounds_crs: Union[CRS, str] = None, + point: Tuple[float, float] = None, + point_crs: Tuple[float, float] = None, + tile: Tuple[int, int, int] = None, + overwrite: bool = False, + multi: int = None, + clip_geometry: str = None, + bidx: List[int] = None, + output_pyramid: str = None, + output_metatiling: int = None, + output_format: str = None, + output_dtype: str = None, + output_geometry_type: str = None, + creation_options: dict = None, + scale_ratio: float = None, + scale_offset: float = None, + resampling_method: str = "nearest", + overviews: bool = False, + overviews_resampling_method: str = "cubic_spline", + cog: bool = False, + vrt: bool = False, + idx_out_dir=None, + msg_callback: Callable = None, + as_iterator: bool = False, +) -> Job: + """ + Convert mapchete outputs or other geodata. + + This is a wrapper around the mapchete.processes.convert process which helps generating tiled + outputs for raster and vector data or single COGs from TileDirectory raster inputs. + + It also supports clipping of the input by a vector dataset. + + If only a subset of a TileDirectory is desired, please see the mapchete.commands.cp command. + + Parameters + ---------- + tiledir : str + Path to TileDirectory or mapchete config. + output : str + Path to output. + zoom : integer or list of integers + Single zoom, minimum and maximum zoom or a list of zoom levels. + area : str, dict, BaseGeometry + Geometry to override bounds or area provided in process configuration. Can be either a + WKT string, a GeoJSON mapping, a shapely geometry or a path to a Fiona-readable file. + area_crs : CRS or str + CRS of area (default: process CRS). + bounds : tuple + Override bounds or area provided in process configuration. + bounds_crs : CRS or str + CRS of area (default: process CRS). + point : iterable + X and y coordinates of point whose corresponding process tile bounds will be used. + point_crs : str or CRS + CRS of point (defaults to process pyramid CRS). + tile : tuple + Zoom, row and column of tile to be processed (cannot be used with zoom) + overwrite : bool + Overwrite existing output. + multi : int + Number of processes used to paralellize tile execution. + clip_geometry : str + Path to Fiona-readable file by which output will be clipped. + bidx : list of integers + Band indexes to read from source. + output_pyramid : str + Output pyramid to write to. + output_metatiling : int + Output metatiling. + output_format : str + Output format. Can be any raster or vector format available by mapchete. + output_dtype : str + Output data type (for raster output only). + output_geometry_type : + Output geometry type (for vector output only). + creation_options : dict + Output driver specific creation options. + scale_ratio : float + Scaling factor (for raster output only). + scale_offset : float + Scaling offset (for raster output only). + resampling_method : str + Resampling method used. (default: nearest). + overviews : bool + Generate overviews (single GTiff output only). + overviews_resampling_method : str + Resampling method used for overviews. (default: cubic_spline) + cog : bool + Write a valid COG. This will automatically generate verviews. (GTiff only) + vrt : bool + Activate VRT index creation. + idx_out_dir : str + Alternative output dir for index. Defaults to TileDirectory path. + msg_callback : Callable + Optional callback function for process messages. + as_iterator : bool + Returns as generator but with a __len__() property. + + Returns + ------- + Job instance either with already processed items or a generator with known length. + + Examples + -------- + >>> execute("foo") + + This will run the whole copy process. + + >>> for i in execute("foo", as_iterator=True): + >>> print(i) + + This will return a generator where through iteration, tiles are copied. + + >>> list(tqdm.tqdm(index("foo", as_iterator=True))) + + Usage within a process bar. + """ + msg_callback = msg_callback or empty_callback + try: + input_info = _get_input_info(tiledir) + output_info = _get_output_info(output) + except Exception as e: + raise ValueError(e) + + # collect mapchete configuration + mapchete_config = dict( + process="mapchete.processes.convert", + input=dict(inp=tiledir, clip=clip_geometry), + pyramid=( + dict( + grid=output_pyramid, + metatiling=( + output_metatiling + or ( + input_info["pyramid"].get("metatiling", 1) + if input_info["pyramid"] + else 1 + ) + ), + pixelbuffer=( + input_info["pyramid"].get("pixelbuffer", 0) + if input_info["pyramid"] + else 0 + ), + ) + if output_pyramid + else input_info["pyramid"] + ), + output=dict( + { + k: v + for k, v in input_info["output_params"].items() + if k not in ["delimiters", "bounds", "mode"] + }, + path=output, + format=( + output_format + or output_info["driver"] + or input_info["output_params"]["format"] + ), + dtype=output_dtype or input_info["output_params"].get("dtype"), + **creation_options, + **dict(overviews=True, overviews_resampling=overviews_resampling_method) + if overviews + else dict(), + ), + config_dir=os.getcwd(), + zoom_levels=zoom or input_info["zoom_levels"], + scale_ratio=scale_ratio, + scale_offset=scale_offset, + resampling=resampling_method, + band_indexes=bidx, + ) + + # assert all required information is there + if mapchete_config["output"]["format"] is None: + # this happens if input file is e.g. JPEG2000 and output is a tile directory + raise ValueError("Output format required.") + if mapchete_config["output"]["format"] == "GTiff": + mapchete_config["output"].update(cog=cog) + output_type = OUTPUT_FORMATS[mapchete_config["output"]["format"]]["data_type"] + if bidx is not None: + mapchete_config["output"].update(bands=len(bidx)) + if mapchete_config["pyramid"] is None: + raise ValueError("Output pyramid required.") + elif mapchete_config["zoom_levels"] is None: + try: + mapchete_config.update( + zoom_levels=dict( + min=0, + max=get_best_zoom_level( + tiledir, mapchete_config["pyramid"]["grid"] + ), + ) + ) + except Exception: + raise ValueError("Zoom levels required.") + elif input_info["input_type"] != output_type: + raise ValueError( + f"Output format type ({output_type}) is incompatible with input format ({input_info['input_type']})." + ) + if output_metatiling: + mapchete_config["output"].update(metatiling=output_metatiling) + if input_info["output_params"].get("schema") and output_geometry_type: + mapchete_config["output"]["schema"].update(geometry=output_geometry_type) + + # determine process bounds + out_pyramid = BufferedTilePyramid.from_dict(mapchete_config["pyramid"]) + inp_bounds = ( + bounds + or reproject_geometry( + box(*input_info["bounds"]), + src_crs=input_info["crs"], + dst_crs=out_pyramid.crs, + ).bounds + if input_info["bounds"] + else out_pyramid.bounds + ) + # if clip-geometry is available, intersect determined bounds with clip bounds + if clip_geometry: + clip_intersection = _clip_bbox( + clip_geometry, dst_crs=out_pyramid.crs + ).intersection(box(*inp_bounds)) + if clip_intersection.is_empty: + msg_callback( + "Process area is empty: clip bounds don't intersect with input bounds." + ) + + def _empty_gen(): + raise StopIteration() + + return Job(iter, [], as_iterator=as_iterator, total=0) + # add process bounds and output type + mapchete_config.update( + bounds=(clip_intersection.bounds if clip_geometry else inp_bounds), + bounds_crs=bounds_crs, + clip_to_output_dtype=mapchete_config["output"].get("dtype", None), + ) + logger.debug(f"temporary config generated: {pformat(mapchete_config)}") + + return execute( + mapchete_config=mapchete_config, + mode="overwrite" if overwrite else "continue", + zoom=zoom, + point=point, + point_crs=point_crs, + bounds=bounds, + bounds_crs=bounds_crs, + area=area, + area_crs=area_crs, + multi=multi or cpu_count(), + vrt=vrt, + idx_out_dir=idx_out_dir, + as_iterator=as_iterator, + msg_callback=msg_callback, + ) + + +def _clip_bbox(clip_geometry, dst_crs=None): + with fiona.open(clip_geometry) as src: + return reproject_geometry(box(*src.bounds), src_crs=src.crs, dst_crs=dst_crs) + + +def _get_input_info(tiledir): + + # assuming single file if path has a file extension + if os.path.splitext(tiledir)[1]: + driver = driver_from_file(tiledir) + + # single file input can be a mapchete file or a rasterio/fiona file + if driver == "Mapchete": + logger.debug("input is mapchete file") + input_info = _input_mapchete_info(tiledir) + + elif driver == "raster_file": + # this should be readable by rasterio + logger.debug("input is raster_file") + input_info = _input_rasterio_info(tiledir) + + elif driver == "vector_file": + # this should be readable by Fiona + input_info = _input_fiona_info(tiledir) + else: # pragma: no cover + raise NotImplementedError(f"driver {driver} is not supported") + + # assuming tile directory + else: + logger.debug("input is tile directory") + input_info = _input_tile_directory_info(tiledir) + + return input_info + + +def _input_mapchete_info(tiledir): + conf = raw_conf(tiledir) + output_params = conf["output"] + pyramid = raw_conf_output_pyramid(conf) + return dict( + output_params=output_params, + pyramid=pyramid.to_dict(), + crs=pyramid.crs, + zoom_levels=validate_zooms(conf["zoom_levels"], expand=False), + pixel_size=None, + input_type=OUTPUT_FORMATS[output_params["format"]]["data_type"], + bounds=conf.get("bounds"), + ) + + +def _input_rasterio_info(tiledir): + with rasterio.open(tiledir) as src: + return dict( + output_params=dict( + bands=src.meta["count"], + dtype=src.meta["dtype"], + format=src.driver if src.driver in available_input_formats() else None, + ), + pyramid=None, + crs=src.crs, + zoom_levels=None, + pixel_size=src.transform[0], + input_type="raster", + bounds=src.bounds, + ) + + +def _input_fiona_info(tiledir): + with fiona.open(tiledir) as src: + return dict( + output_params=dict( + schema=src.schema, + format=src.driver if src.driver in available_input_formats() else None, + ), + pyramid=None, + crs=src.crs, + zoom_levels=None, + input_type="vector", + bounds=src.bounds, + ) + + +def _input_tile_directory_info(tiledir): + conf = read_json(os.path.join(tiledir, "metadata.json")) + pyramid = BufferedTilePyramid.from_dict(conf["pyramid"]) + return dict( + output_params=conf["driver"], + pyramid=pyramid.to_dict(), + crs=pyramid.crs, + zoom_levels=None, + pixel_size=None, + input_type=OUTPUT_FORMATS[conf["driver"]["format"]]["data_type"], + bounds=None, + ) + + +def _get_output_info(output): + _, file_ext = os.path.splitext(output) + if not file_ext: + return dict(type="TileDirectory", driver=None) + elif file_ext == ".tif": + return dict(type="SingleFile", driver="GTiff") + else: + raise TypeError(f"Could not determine output from extension: {file_ext}") diff --git a/mapchete/commands/_execute.py b/mapchete/commands/_execute.py new file mode 100755 index 00000000..6b0324d8 --- /dev/null +++ b/mapchete/commands/_execute.py @@ -0,0 +1,155 @@ +import logging +from multiprocessing import cpu_count +from rasterio.crs import CRS +from shapely.geometry.base import BaseGeometry +from typing import Callable, List, Tuple, Union + +import mapchete +from mapchete.commands._job import empty_callback, Job +from mapchete.config import bounds_from_opts, raw_conf, raw_conf_process_pyramid + +logger = logging.getLogger(__name__) + + +def execute( + mapchete_config: Union[str, dict], + zoom: Union[int, List[int]] = None, + area: Union[BaseGeometry, str, dict] = None, + area_crs: Union[CRS, str] = None, + bounds: Tuple[float] = None, + bounds_crs: Union[CRS, str] = None, + point: Tuple[float, float] = None, + point_crs: Tuple[float, float] = None, + tile: Tuple[int, int, int] = None, + overwrite: bool = False, + mode: str = "continue", + multi: int = None, + max_chunksize: int = None, + multiprocessing_start_method: str = None, + vrt: bool = False, + idx_out_dir: str = None, + msg_callback: Callable = None, + as_iterator: bool = False, +) -> Job: + """ + Execute a Mapchete process. + + Parameters + ---------- + mapchete_config : str or dict + Mapchete configuration as file path or dictionary. + zoom : integer or list of integers + Single zoom, minimum and maximum zoom or a list of zoom levels. + area : str, dict, BaseGeometry + Geometry to override bounds or area provided in process configuration. Can be either a + WKT string, a GeoJSON mapping, a shapely geometry or a path to a Fiona-readable file. + area_crs : CRS or str + CRS of area (default: process CRS). + bounds : tuple + Override bounds or area provided in process configuration. + bounds_crs : CRS or str + CRS of area (default: process CRS). + point : iterable + X and y coordinates of point whose corresponding process tile bounds will be used. + point_crs : str or CRS + CRS of point (defaults to process pyramid CRS). + tile : tuple + Zoom, row and column of tile to be processed (cannot be used with zoom) + overwrite : bool + Overwrite existing output. + mode : str + Set process mode. One of "readonly", "continue" or "overwrite". + multi : int + Number of processes used to paralellize tile execution. + max_chunksize : int + Maximum number of process tiles to be queued for each worker. (default: 1) + multiprocessing_start_method : str + Method used by multiprocessing module to start child workers. Availability of methods + depends on OS. + vrt : bool + Activate VRT index creation. + idx_out_dir : str + Alternative output dir for index. Defaults to TileDirectory path. + msg_callback : Callable + Optional callback function for process messages. + as_iterator : bool + Returns as generator but with a __len__() property. + + Returns + ------- + Job instance either with already processed items or a generator with known length. + + Examples + -------- + >>> execute("foo") + + This will run the whole copy process. + + >>> for i in execute("foo", as_iterator=True): + >>> print(i) + + This will return a generator where through iteration, tiles are copied. + + >>> list(tqdm.tqdm(index("foo", as_iterator=True))) + + Usage within a process bar. + """ + mode = "overwrite" if overwrite else mode + msg_callback = msg_callback or empty_callback + multi = multi or cpu_count() + + if tile: + tile = raw_conf_process_pyramid(raw_conf(mapchete_config)).tile(*tile) + bounds = tile.bounds + zoom = tile.zoom + else: + bounds = bounds_from_opts( + point=point, + point_crs=point_crs, + bounds=bounds, + bounds_crs=bounds_crs, + raw_conf=raw_conf(mapchete_config), + ) + + # be careful opening mapchete not as context manager + mp = mapchete.open( + mapchete_config, + mode=mode, + bounds=bounds, + zoom=zoom, + area=area, + area_crs=area_crs, + ) + try: + tiles_count = mp.count_tiles() + if tile: + msg_callback("processing 1 tile") + else: + msg_callback(f"processing {tiles_count} tile(s) on {multi} worker(s)") + return Job( + _msg_wrapper, + msg_callback, + mp, + tile=tile, + multi=multi, + zoom=None if tile else zoom, + max_chunksize=max_chunksize, + multiprocessing_start_method=multiprocessing_start_method, + as_iterator=as_iterator, + total=1 if tile else tiles_count, + ) + # explicitly exit the mp object on failure + except Exception: + mp.__exit__(None, None, None) + + +def _msg_wrapper(msg_callback, mp, **kwargs): + try: + for process_info in mp.batch_processor(**kwargs): + yield process_info + msg_callback( + f"Tile {process_info.tile.id}: {process_info.process_msg}, {process_info.write_msg}" + ) + # explicitly exit the mp object on success + finally: + mp.__exit__(None, None, None) diff --git a/mapchete/commands/_index.py b/mapchete/commands/_index.py index 18441dd3..dfdec483 100644 --- a/mapchete/commands/_index.py +++ b/mapchete/commands/_index.py @@ -1,5 +1,3 @@ -"""Create index for process output.""" - import logging import os from rasterio.crs import CRS @@ -38,6 +36,7 @@ def index( fs_opts: dict = None, msg_callback: Callable = None, as_iterator: bool = False, + **kwargs, ) -> Job: """ Create one or more indexes from a TileDirectory. diff --git a/test/test_cli.py b/test/test_cli.py index 88697cb7..aa6e7d40 100644 --- a/test/test_cli.py +++ b/test/test_cli.py @@ -41,9 +41,13 @@ def version_is_greater_equal(a, b): def run_cli(args, expected_exit_code=0, output_contains=None, raise_exc=True): - result = CliRunner(env=dict(MAPCHETE_TEST="TRUE")).invoke(mapchete_cli, args) + result = CliRunner(env=dict(MAPCHETE_TEST="TRUE"), mix_stderr=True).invoke( + mapchete_cli, args + ) if output_contains: - assert output_contains in result.output + assert output_contains in result.output or output_contains in str( + result.exception + ) if raise_exc and result.exception: raise result.exception assert result.exit_code == expected_exit_code @@ -69,16 +73,6 @@ def test_main(mp_tmpdir): ) -def test_missing_input_file(mp_tmpdir): - """Check if IOError is raised if input-file is invalid.""" - run_cli( - ["execute", "process.mapchete", "--input-file", "invalid.tif"], - expected_exit_code=2, - output_contains="Path 'process.mapchete' does not exist.", - raise_exc=False, - ) - - def test_create_and_execute(mp_tmpdir, cleantopo_br_tif): """Run mapchete create and execute.""" temp_mapchete = os.path.join(mp_tmpdir, "temp.mapchete") @@ -102,22 +96,6 @@ def test_create_and_execute(mp_tmpdir, cleantopo_br_tif): config["output"].update(bands=1, dtype="uint8", path=mp_tmpdir) with open(temp_mapchete, "w") as config_file: config_file.write(yaml.dump(config, default_flow_style=False)) - # run process for single tile, this creates an empty output error - with pytest.raises(MapcheteProcessOutputError): - run_cli( - [ - "execute", - temp_mapchete, - "--tile", - "6", - "62", - "124", - "--input-file", - cleantopo_br_tif, - "-d", - ], - expected_exit_code=-1, - ) def test_create_existing(mp_tmpdir): @@ -170,8 +148,6 @@ def test_execute_multiprocessing(mp_tmpdir, cleantopo_br, cleantopo_br_tif): temp_mapchete, "--zoom", "5", - "--input-file", - cleantopo_br_tif, "-m", "2", "-d", @@ -235,11 +211,6 @@ def test_execute_logfile(mp_tmpdir, example_mapchete): assert "DEBUG" in log.read() -def test_execute_wkt_bounds(mp_tmpdir, example_mapchete, wkt_geom): - """Using bounds from WKT.""" - run_cli(["execute", example_mapchete.path, "--wkt-geometry", wkt_geom]) - - def test_execute_wkt_area(mp_tmpdir, example_mapchete, wkt_geom): """Using area from WKT.""" run_cli(["execute", example_mapchete.path, "--area", wkt_geom]) @@ -518,7 +489,7 @@ def test_convert_clip(cleantopo_br_tif, mp_tmpdir, landpoly): "geodetic", "--clip-geometry", landpoly, - "-d", + "-v", ], output_contains="Process area is empty", ) @@ -744,7 +715,7 @@ def test_convert_errors(s2_band_jp2, mp_tmpdir, s2_band, cleantopo_br, landpoly) # output format required run_cli( ["convert", s2_band_jp2, mp_tmpdir, "--output-pyramid", "geodetic"], - expected_exit_code=2, + expected_exit_code=1, output_contains="Output format required.", raise_exc=False, ) @@ -752,7 +723,7 @@ def test_convert_errors(s2_band_jp2, mp_tmpdir, s2_band, cleantopo_br, landpoly) # output pyramid reqired run_cli( ["convert", s2_band, mp_tmpdir], - expected_exit_code=2, + expected_exit_code=1, output_contains="Output pyramid required.", raise_exc=False, ) @@ -773,7 +744,7 @@ def test_convert_errors(s2_band_jp2, mp_tmpdir, s2_band, cleantopo_br, landpoly) "--output-pyramid", "geodetic", ], - expected_exit_code=2, + expected_exit_code=1, output_contains="Zoom levels required.", raise_exc=False, ) @@ -791,7 +762,7 @@ def test_convert_errors(s2_band_jp2, mp_tmpdir, s2_band, cleantopo_br, landpoly) "--output-format", "GeoJSON", ], - expected_exit_code=2, + expected_exit_code=1, output_contains=( "Output format type (vector) is incompatible with input format (raster)." ), @@ -809,7 +780,7 @@ def test_convert_errors(s2_band_jp2, mp_tmpdir, s2_band, cleantopo_br, landpoly) "--zoom", "5", ], - expected_exit_code=2, + expected_exit_code=1, output_contains=("Could not determine output from extension"), raise_exc=False, ) @@ -838,7 +809,6 @@ def test_serve_cli_params(cleantopo_br, mp_tmpdir): ["serve", cleantopo_br.path, "--overwrite"], ["serve", cleantopo_br.path, "--readonly"], ["serve", cleantopo_br.path, "--memory"], - ["serve", cleantopo_br.path, "--input-file", cleantopo_br.path], ]: run_cli(args) @@ -990,21 +960,6 @@ def test_index_geojson_tile(mp_tmpdir, cleantopo_tl): assert len(list(src)) == 1 -def test_index_geojson_wkt_geom(mp_tmpdir, cleantopo_br, wkt_geom): - # execute process at zoom 3 - run_cli(["execute", cleantopo_br.path, "--debug", "--wkt-geometry", wkt_geom]) - - # generate index for zoom 3 - run_cli( - ["index", cleantopo_br.path, "--geojson", "--debug", "--wkt-geometry", wkt_geom] - ) - - with mapchete.open(cleantopo_br.dict) as mp: - files = os.listdir(mp.config.output.path) - assert len(files) == 7 - assert "3.geojson" in files - - def test_index_geojson_wkt_area(mp_tmpdir, cleantopo_br, wkt_geom): # execute process at zoom 3 run_cli(["execute", cleantopo_br.path, "--debug", "--area", wkt_geom]) @@ -1096,7 +1051,7 @@ def test_index_text(cleantopo_br): def test_index_errors(mp_tmpdir, cleantopo_br): - with pytest.raises(SystemExit): + with pytest.raises(ValueError): run_cli(["index", cleantopo_br.path, "-z", "5", "--debug"]) with pytest.raises(SystemExit): From 23f7a60237f8575498f235739e1420e1a6089511 Mon Sep 17 00:00:00 2001 From: Joachim Ungar Date: Fri, 16 Jul 2021 18:59:21 +0200 Subject: [PATCH 6/9] update docs --- .../apidoc/mapchete.cli.default.convert.rst | 6 +++--- doc/source/apidoc/mapchete.cli.default.cp.rst | 7 +++++++ .../apidoc/mapchete.cli.default.create.rst | 6 +++--- .../apidoc/mapchete.cli.default.execute.rst | 6 +++--- .../apidoc/mapchete.cli.default.formats.rst | 6 +++--- .../apidoc/mapchete.cli.default.index.rst | 6 +++--- .../apidoc/mapchete.cli.default.processes.rst | 6 +++--- doc/source/apidoc/mapchete.cli.default.rm.rst | 7 +++++++ doc/source/apidoc/mapchete.cli.default.rst | 9 ++++++--- .../apidoc/mapchete.cli.default.serve.rst | 6 +++--- doc/source/apidoc/mapchete.cli.main.rst | 6 +++--- doc/source/apidoc/mapchete.cli.rst | 10 ++++++---- doc/source/apidoc/mapchete.cli.utils.rst | 6 +++--- doc/source/apidoc/mapchete.commands.rst | 10 ++++++++++ doc/source/apidoc/mapchete.commons.clip.rst | 6 +++--- .../apidoc/mapchete.commons.contours.rst | 6 +++--- .../apidoc/mapchete.commons.hillshade.rst | 6 +++--- doc/source/apidoc/mapchete.commons.rst | 7 ++++--- doc/source/apidoc/mapchete.config.rst | 6 +++--- doc/source/apidoc/mapchete.errors.rst | 6 +++--- doc/source/apidoc/mapchete.formats.base.rst | 6 +++--- .../mapchete.formats.default.flatgeobuf.rst | 7 +++++++ .../mapchete.formats.default.geobuf.rst | 7 +++++++ .../mapchete.formats.default.geojson.rst | 6 +++--- .../apidoc/mapchete.formats.default.gtiff.rst | 6 +++--- ...apchete.formats.default.mapchete_input.rst | 6 +++--- .../apidoc/mapchete.formats.default.png.rst | 6 +++--- ...mapchete.formats.default.png_hillshade.rst | 6 +++--- .../mapchete.formats.default.raster_file.rst | 6 +++--- .../apidoc/mapchete.formats.default.rst | 9 ++++++--- ...apchete.formats.default.tile_directory.rst | 6 +++--- .../mapchete.formats.default.vector_file.rst | 6 +++--- .../apidoc/mapchete.formats.drivers.rst | 6 +++--- doc/source/apidoc/mapchete.formats.rst | 10 ++++++---- doc/source/apidoc/mapchete.index.rst | 6 +++--- doc/source/apidoc/mapchete.io.raster.rst | 6 +++--- doc/source/apidoc/mapchete.io.rst | 7 ++++--- doc/source/apidoc/mapchete.io.vector.rst | 6 +++--- doc/source/apidoc/mapchete.log.rst | 6 +++--- .../apidoc/mapchete.processes.contours.rst | 6 +++--- .../apidoc/mapchete.processes.convert.rst | 6 +++--- ...ete.processes.examples.example_process.rst | 6 +++--- .../apidoc/mapchete.processes.examples.rst | 7 ++++--- .../apidoc/mapchete.processes.hillshade.rst | 6 +++--- doc/source/apidoc/mapchete.processes.rst | 10 ++++++---- doc/source/apidoc/mapchete.rst | 19 +++++++++++-------- doc/source/apidoc/mapchete.tile.rst | 6 +++--- doc/source/apidoc/mapchete.validate.rst | 6 +++--- mapchete/commands/_convert.py | 8 ++++---- mapchete/commands/_execute.py | 4 ++-- mapchete/commands/_index.py | 2 +- 51 files changed, 200 insertions(+), 144 deletions(-) create mode 100644 doc/source/apidoc/mapchete.cli.default.cp.rst create mode 100644 doc/source/apidoc/mapchete.cli.default.rm.rst create mode 100644 doc/source/apidoc/mapchete.commands.rst create mode 100644 doc/source/apidoc/mapchete.formats.default.flatgeobuf.rst create mode 100644 doc/source/apidoc/mapchete.formats.default.geobuf.rst diff --git a/doc/source/apidoc/mapchete.cli.default.convert.rst b/doc/source/apidoc/mapchete.cli.default.convert.rst index 84c68f1d..99eb8eca 100644 --- a/doc/source/apidoc/mapchete.cli.default.convert.rst +++ b/doc/source/apidoc/mapchete.cli.default.convert.rst @@ -2,6 +2,6 @@ mapchete.cli.default.convert module =================================== .. automodule:: mapchete.cli.default.convert - :members: - :undoc-members: - :show-inheritance: + :members: + :undoc-members: + :show-inheritance: diff --git a/doc/source/apidoc/mapchete.cli.default.cp.rst b/doc/source/apidoc/mapchete.cli.default.cp.rst new file mode 100644 index 00000000..29fd3e37 --- /dev/null +++ b/doc/source/apidoc/mapchete.cli.default.cp.rst @@ -0,0 +1,7 @@ +mapchete.cli.default.cp module +============================== + +.. automodule:: mapchete.cli.default.cp + :members: + :undoc-members: + :show-inheritance: diff --git a/doc/source/apidoc/mapchete.cli.default.create.rst b/doc/source/apidoc/mapchete.cli.default.create.rst index 7a9ff23d..856efe13 100644 --- a/doc/source/apidoc/mapchete.cli.default.create.rst +++ b/doc/source/apidoc/mapchete.cli.default.create.rst @@ -2,6 +2,6 @@ mapchete.cli.default.create module ================================== .. automodule:: mapchete.cli.default.create - :members: - :undoc-members: - :show-inheritance: + :members: + :undoc-members: + :show-inheritance: diff --git a/doc/source/apidoc/mapchete.cli.default.execute.rst b/doc/source/apidoc/mapchete.cli.default.execute.rst index 2777b362..434167c7 100644 --- a/doc/source/apidoc/mapchete.cli.default.execute.rst +++ b/doc/source/apidoc/mapchete.cli.default.execute.rst @@ -2,6 +2,6 @@ mapchete.cli.default.execute module =================================== .. automodule:: mapchete.cli.default.execute - :members: - :undoc-members: - :show-inheritance: + :members: + :undoc-members: + :show-inheritance: diff --git a/doc/source/apidoc/mapchete.cli.default.formats.rst b/doc/source/apidoc/mapchete.cli.default.formats.rst index 6c23f340..df63b832 100644 --- a/doc/source/apidoc/mapchete.cli.default.formats.rst +++ b/doc/source/apidoc/mapchete.cli.default.formats.rst @@ -2,6 +2,6 @@ mapchete.cli.default.formats module =================================== .. automodule:: mapchete.cli.default.formats - :members: - :undoc-members: - :show-inheritance: + :members: + :undoc-members: + :show-inheritance: diff --git a/doc/source/apidoc/mapchete.cli.default.index.rst b/doc/source/apidoc/mapchete.cli.default.index.rst index cdd1d74d..e20cf940 100644 --- a/doc/source/apidoc/mapchete.cli.default.index.rst +++ b/doc/source/apidoc/mapchete.cli.default.index.rst @@ -2,6 +2,6 @@ mapchete.cli.default.index module ================================= .. automodule:: mapchete.cli.default.index - :members: - :undoc-members: - :show-inheritance: + :members: + :undoc-members: + :show-inheritance: diff --git a/doc/source/apidoc/mapchete.cli.default.processes.rst b/doc/source/apidoc/mapchete.cli.default.processes.rst index fd9809ff..681847d0 100644 --- a/doc/source/apidoc/mapchete.cli.default.processes.rst +++ b/doc/source/apidoc/mapchete.cli.default.processes.rst @@ -2,6 +2,6 @@ mapchete.cli.default.processes module ===================================== .. automodule:: mapchete.cli.default.processes - :members: - :undoc-members: - :show-inheritance: + :members: + :undoc-members: + :show-inheritance: diff --git a/doc/source/apidoc/mapchete.cli.default.rm.rst b/doc/source/apidoc/mapchete.cli.default.rm.rst new file mode 100644 index 00000000..73820fdc --- /dev/null +++ b/doc/source/apidoc/mapchete.cli.default.rm.rst @@ -0,0 +1,7 @@ +mapchete.cli.default.rm module +============================== + +.. automodule:: mapchete.cli.default.rm + :members: + :undoc-members: + :show-inheritance: diff --git a/doc/source/apidoc/mapchete.cli.default.rst b/doc/source/apidoc/mapchete.cli.default.rst index 3363bb6f..31ee79db 100644 --- a/doc/source/apidoc/mapchete.cli.default.rst +++ b/doc/source/apidoc/mapchete.cli.default.rst @@ -5,19 +5,22 @@ Submodules ---------- .. toctree:: + :maxdepth: 4 mapchete.cli.default.convert + mapchete.cli.default.cp mapchete.cli.default.create mapchete.cli.default.execute mapchete.cli.default.formats mapchete.cli.default.index mapchete.cli.default.processes + mapchete.cli.default.rm mapchete.cli.default.serve Module contents --------------- .. automodule:: mapchete.cli.default - :members: - :undoc-members: - :show-inheritance: + :members: + :undoc-members: + :show-inheritance: diff --git a/doc/source/apidoc/mapchete.cli.default.serve.rst b/doc/source/apidoc/mapchete.cli.default.serve.rst index 75c55859..f456ca98 100644 --- a/doc/source/apidoc/mapchete.cli.default.serve.rst +++ b/doc/source/apidoc/mapchete.cli.default.serve.rst @@ -2,6 +2,6 @@ mapchete.cli.default.serve module ================================= .. automodule:: mapchete.cli.default.serve - :members: - :undoc-members: - :show-inheritance: + :members: + :undoc-members: + :show-inheritance: diff --git a/doc/source/apidoc/mapchete.cli.main.rst b/doc/source/apidoc/mapchete.cli.main.rst index ffceb2bb..53f9b791 100644 --- a/doc/source/apidoc/mapchete.cli.main.rst +++ b/doc/source/apidoc/mapchete.cli.main.rst @@ -2,6 +2,6 @@ mapchete.cli.main module ======================== .. automodule:: mapchete.cli.main - :members: - :undoc-members: - :show-inheritance: + :members: + :undoc-members: + :show-inheritance: diff --git a/doc/source/apidoc/mapchete.cli.rst b/doc/source/apidoc/mapchete.cli.rst index 0e976aba..e628f73f 100644 --- a/doc/source/apidoc/mapchete.cli.rst +++ b/doc/source/apidoc/mapchete.cli.rst @@ -5,13 +5,15 @@ Subpackages ----------- .. toctree:: + :maxdepth: 4 - mapchete.cli.default + mapchete.cli.default Submodules ---------- .. toctree:: + :maxdepth: 4 mapchete.cli.main mapchete.cli.utils @@ -20,6 +22,6 @@ Module contents --------------- .. automodule:: mapchete.cli - :members: - :undoc-members: - :show-inheritance: + :members: + :undoc-members: + :show-inheritance: diff --git a/doc/source/apidoc/mapchete.cli.utils.rst b/doc/source/apidoc/mapchete.cli.utils.rst index 075f5fe3..a1c7c7f7 100644 --- a/doc/source/apidoc/mapchete.cli.utils.rst +++ b/doc/source/apidoc/mapchete.cli.utils.rst @@ -2,6 +2,6 @@ mapchete.cli.utils module ========================= .. automodule:: mapchete.cli.utils - :members: - :undoc-members: - :show-inheritance: + :members: + :undoc-members: + :show-inheritance: diff --git a/doc/source/apidoc/mapchete.commands.rst b/doc/source/apidoc/mapchete.commands.rst new file mode 100644 index 00000000..e0bbc757 --- /dev/null +++ b/doc/source/apidoc/mapchete.commands.rst @@ -0,0 +1,10 @@ +mapchete.commands package +========================= + +Module contents +--------------- + +.. automodule:: mapchete.commands + :members: + :undoc-members: + :show-inheritance: diff --git a/doc/source/apidoc/mapchete.commons.clip.rst b/doc/source/apidoc/mapchete.commons.clip.rst index adfed0c2..a6d0ab1a 100644 --- a/doc/source/apidoc/mapchete.commons.clip.rst +++ b/doc/source/apidoc/mapchete.commons.clip.rst @@ -2,6 +2,6 @@ mapchete.commons.clip module ============================ .. automodule:: mapchete.commons.clip - :members: - :undoc-members: - :show-inheritance: + :members: + :undoc-members: + :show-inheritance: diff --git a/doc/source/apidoc/mapchete.commons.contours.rst b/doc/source/apidoc/mapchete.commons.contours.rst index 56fb50dd..5092f78a 100644 --- a/doc/source/apidoc/mapchete.commons.contours.rst +++ b/doc/source/apidoc/mapchete.commons.contours.rst @@ -2,6 +2,6 @@ mapchete.commons.contours module ================================ .. automodule:: mapchete.commons.contours - :members: - :undoc-members: - :show-inheritance: + :members: + :undoc-members: + :show-inheritance: diff --git a/doc/source/apidoc/mapchete.commons.hillshade.rst b/doc/source/apidoc/mapchete.commons.hillshade.rst index 38f9b3c0..002b06ce 100644 --- a/doc/source/apidoc/mapchete.commons.hillshade.rst +++ b/doc/source/apidoc/mapchete.commons.hillshade.rst @@ -2,6 +2,6 @@ mapchete.commons.hillshade module ================================= .. automodule:: mapchete.commons.hillshade - :members: - :undoc-members: - :show-inheritance: + :members: + :undoc-members: + :show-inheritance: diff --git a/doc/source/apidoc/mapchete.commons.rst b/doc/source/apidoc/mapchete.commons.rst index 081bd4b2..891673ef 100644 --- a/doc/source/apidoc/mapchete.commons.rst +++ b/doc/source/apidoc/mapchete.commons.rst @@ -5,6 +5,7 @@ Submodules ---------- .. toctree:: + :maxdepth: 4 mapchete.commons.clip mapchete.commons.contours @@ -14,6 +15,6 @@ Module contents --------------- .. automodule:: mapchete.commons - :members: - :undoc-members: - :show-inheritance: + :members: + :undoc-members: + :show-inheritance: diff --git a/doc/source/apidoc/mapchete.config.rst b/doc/source/apidoc/mapchete.config.rst index 8ef00a47..1e0562d2 100644 --- a/doc/source/apidoc/mapchete.config.rst +++ b/doc/source/apidoc/mapchete.config.rst @@ -2,6 +2,6 @@ mapchete.config module ====================== .. automodule:: mapchete.config - :members: - :undoc-members: - :show-inheritance: + :members: + :undoc-members: + :show-inheritance: diff --git a/doc/source/apidoc/mapchete.errors.rst b/doc/source/apidoc/mapchete.errors.rst index 1947953e..e0d0e2bd 100644 --- a/doc/source/apidoc/mapchete.errors.rst +++ b/doc/source/apidoc/mapchete.errors.rst @@ -2,6 +2,6 @@ mapchete.errors module ====================== .. automodule:: mapchete.errors - :members: - :undoc-members: - :show-inheritance: + :members: + :undoc-members: + :show-inheritance: diff --git a/doc/source/apidoc/mapchete.formats.base.rst b/doc/source/apidoc/mapchete.formats.base.rst index 0baf06f4..0047faf3 100644 --- a/doc/source/apidoc/mapchete.formats.base.rst +++ b/doc/source/apidoc/mapchete.formats.base.rst @@ -2,6 +2,6 @@ mapchete.formats.base module ============================ .. automodule:: mapchete.formats.base - :members: - :undoc-members: - :show-inheritance: + :members: + :undoc-members: + :show-inheritance: diff --git a/doc/source/apidoc/mapchete.formats.default.flatgeobuf.rst b/doc/source/apidoc/mapchete.formats.default.flatgeobuf.rst new file mode 100644 index 00000000..ac42b5ae --- /dev/null +++ b/doc/source/apidoc/mapchete.formats.default.flatgeobuf.rst @@ -0,0 +1,7 @@ +mapchete.formats.default.flatgeobuf module +========================================== + +.. automodule:: mapchete.formats.default.flatgeobuf + :members: + :undoc-members: + :show-inheritance: diff --git a/doc/source/apidoc/mapchete.formats.default.geobuf.rst b/doc/source/apidoc/mapchete.formats.default.geobuf.rst new file mode 100644 index 00000000..70588a24 --- /dev/null +++ b/doc/source/apidoc/mapchete.formats.default.geobuf.rst @@ -0,0 +1,7 @@ +mapchete.formats.default.geobuf module +====================================== + +.. automodule:: mapchete.formats.default.geobuf + :members: + :undoc-members: + :show-inheritance: diff --git a/doc/source/apidoc/mapchete.formats.default.geojson.rst b/doc/source/apidoc/mapchete.formats.default.geojson.rst index b1b20d42..9c0e6111 100644 --- a/doc/source/apidoc/mapchete.formats.default.geojson.rst +++ b/doc/source/apidoc/mapchete.formats.default.geojson.rst @@ -2,6 +2,6 @@ mapchete.formats.default.geojson module ======================================= .. automodule:: mapchete.formats.default.geojson - :members: - :undoc-members: - :show-inheritance: + :members: + :undoc-members: + :show-inheritance: diff --git a/doc/source/apidoc/mapchete.formats.default.gtiff.rst b/doc/source/apidoc/mapchete.formats.default.gtiff.rst index 49b66a18..d6907f9d 100644 --- a/doc/source/apidoc/mapchete.formats.default.gtiff.rst +++ b/doc/source/apidoc/mapchete.formats.default.gtiff.rst @@ -2,6 +2,6 @@ mapchete.formats.default.gtiff module ===================================== .. automodule:: mapchete.formats.default.gtiff - :members: - :undoc-members: - :show-inheritance: + :members: + :undoc-members: + :show-inheritance: diff --git a/doc/source/apidoc/mapchete.formats.default.mapchete_input.rst b/doc/source/apidoc/mapchete.formats.default.mapchete_input.rst index 5c572a76..11d84fab 100644 --- a/doc/source/apidoc/mapchete.formats.default.mapchete_input.rst +++ b/doc/source/apidoc/mapchete.formats.default.mapchete_input.rst @@ -2,6 +2,6 @@ mapchete.formats.default.mapchete\_input module =============================================== .. automodule:: mapchete.formats.default.mapchete_input - :members: - :undoc-members: - :show-inheritance: + :members: + :undoc-members: + :show-inheritance: diff --git a/doc/source/apidoc/mapchete.formats.default.png.rst b/doc/source/apidoc/mapchete.formats.default.png.rst index dc4358e7..9bf633c6 100644 --- a/doc/source/apidoc/mapchete.formats.default.png.rst +++ b/doc/source/apidoc/mapchete.formats.default.png.rst @@ -2,6 +2,6 @@ mapchete.formats.default.png module =================================== .. automodule:: mapchete.formats.default.png - :members: - :undoc-members: - :show-inheritance: + :members: + :undoc-members: + :show-inheritance: diff --git a/doc/source/apidoc/mapchete.formats.default.png_hillshade.rst b/doc/source/apidoc/mapchete.formats.default.png_hillshade.rst index e5f2aca9..ad0bc7b1 100644 --- a/doc/source/apidoc/mapchete.formats.default.png_hillshade.rst +++ b/doc/source/apidoc/mapchete.formats.default.png_hillshade.rst @@ -2,6 +2,6 @@ mapchete.formats.default.png\_hillshade module ============================================== .. automodule:: mapchete.formats.default.png_hillshade - :members: - :undoc-members: - :show-inheritance: + :members: + :undoc-members: + :show-inheritance: diff --git a/doc/source/apidoc/mapchete.formats.default.raster_file.rst b/doc/source/apidoc/mapchete.formats.default.raster_file.rst index fa66bb8d..4cea8760 100644 --- a/doc/source/apidoc/mapchete.formats.default.raster_file.rst +++ b/doc/source/apidoc/mapchete.formats.default.raster_file.rst @@ -2,6 +2,6 @@ mapchete.formats.default.raster\_file module ============================================ .. automodule:: mapchete.formats.default.raster_file - :members: - :undoc-members: - :show-inheritance: + :members: + :undoc-members: + :show-inheritance: diff --git a/doc/source/apidoc/mapchete.formats.default.rst b/doc/source/apidoc/mapchete.formats.default.rst index 413052b5..acac96d6 100644 --- a/doc/source/apidoc/mapchete.formats.default.rst +++ b/doc/source/apidoc/mapchete.formats.default.rst @@ -5,7 +5,10 @@ Submodules ---------- .. toctree:: + :maxdepth: 4 + mapchete.formats.default.flatgeobuf + mapchete.formats.default.geobuf mapchete.formats.default.geojson mapchete.formats.default.gtiff mapchete.formats.default.mapchete_input @@ -19,6 +22,6 @@ Module contents --------------- .. automodule:: mapchete.formats.default - :members: - :undoc-members: - :show-inheritance: + :members: + :undoc-members: + :show-inheritance: diff --git a/doc/source/apidoc/mapchete.formats.default.tile_directory.rst b/doc/source/apidoc/mapchete.formats.default.tile_directory.rst index 3867c687..4b83da4c 100644 --- a/doc/source/apidoc/mapchete.formats.default.tile_directory.rst +++ b/doc/source/apidoc/mapchete.formats.default.tile_directory.rst @@ -2,6 +2,6 @@ mapchete.formats.default.tile\_directory module =============================================== .. automodule:: mapchete.formats.default.tile_directory - :members: - :undoc-members: - :show-inheritance: + :members: + :undoc-members: + :show-inheritance: diff --git a/doc/source/apidoc/mapchete.formats.default.vector_file.rst b/doc/source/apidoc/mapchete.formats.default.vector_file.rst index d18b36f1..d25edb3e 100644 --- a/doc/source/apidoc/mapchete.formats.default.vector_file.rst +++ b/doc/source/apidoc/mapchete.formats.default.vector_file.rst @@ -2,6 +2,6 @@ mapchete.formats.default.vector\_file module ============================================ .. automodule:: mapchete.formats.default.vector_file - :members: - :undoc-members: - :show-inheritance: + :members: + :undoc-members: + :show-inheritance: diff --git a/doc/source/apidoc/mapchete.formats.drivers.rst b/doc/source/apidoc/mapchete.formats.drivers.rst index 97a4989c..d3da5074 100644 --- a/doc/source/apidoc/mapchete.formats.drivers.rst +++ b/doc/source/apidoc/mapchete.formats.drivers.rst @@ -2,6 +2,6 @@ mapchete.formats.drivers module =============================== .. automodule:: mapchete.formats.drivers - :members: - :undoc-members: - :show-inheritance: + :members: + :undoc-members: + :show-inheritance: diff --git a/doc/source/apidoc/mapchete.formats.rst b/doc/source/apidoc/mapchete.formats.rst index 393d555c..0bb40123 100644 --- a/doc/source/apidoc/mapchete.formats.rst +++ b/doc/source/apidoc/mapchete.formats.rst @@ -5,13 +5,15 @@ Subpackages ----------- .. toctree:: + :maxdepth: 4 - mapchete.formats.default + mapchete.formats.default Submodules ---------- .. toctree:: + :maxdepth: 4 mapchete.formats.base mapchete.formats.drivers @@ -20,6 +22,6 @@ Module contents --------------- .. automodule:: mapchete.formats - :members: - :undoc-members: - :show-inheritance: + :members: + :undoc-members: + :show-inheritance: diff --git a/doc/source/apidoc/mapchete.index.rst b/doc/source/apidoc/mapchete.index.rst index f9c179ce..4f966b89 100644 --- a/doc/source/apidoc/mapchete.index.rst +++ b/doc/source/apidoc/mapchete.index.rst @@ -2,6 +2,6 @@ mapchete.index module ===================== .. automodule:: mapchete.index - :members: - :undoc-members: - :show-inheritance: + :members: + :undoc-members: + :show-inheritance: diff --git a/doc/source/apidoc/mapchete.io.raster.rst b/doc/source/apidoc/mapchete.io.raster.rst index 20c491d6..8071613f 100644 --- a/doc/source/apidoc/mapchete.io.raster.rst +++ b/doc/source/apidoc/mapchete.io.raster.rst @@ -2,6 +2,6 @@ mapchete.io.raster module ========================= .. automodule:: mapchete.io.raster - :members: - :undoc-members: - :show-inheritance: + :members: + :undoc-members: + :show-inheritance: diff --git a/doc/source/apidoc/mapchete.io.rst b/doc/source/apidoc/mapchete.io.rst index c6b4a279..02ba1764 100644 --- a/doc/source/apidoc/mapchete.io.rst +++ b/doc/source/apidoc/mapchete.io.rst @@ -5,6 +5,7 @@ Submodules ---------- .. toctree:: + :maxdepth: 4 mapchete.io.raster mapchete.io.vector @@ -13,6 +14,6 @@ Module contents --------------- .. automodule:: mapchete.io - :members: - :undoc-members: - :show-inheritance: + :members: + :undoc-members: + :show-inheritance: diff --git a/doc/source/apidoc/mapchete.io.vector.rst b/doc/source/apidoc/mapchete.io.vector.rst index 7ae78eaa..a2113ee6 100644 --- a/doc/source/apidoc/mapchete.io.vector.rst +++ b/doc/source/apidoc/mapchete.io.vector.rst @@ -2,6 +2,6 @@ mapchete.io.vector module ========================= .. automodule:: mapchete.io.vector - :members: - :undoc-members: - :show-inheritance: + :members: + :undoc-members: + :show-inheritance: diff --git a/doc/source/apidoc/mapchete.log.rst b/doc/source/apidoc/mapchete.log.rst index 36bf698f..4cd4f933 100644 --- a/doc/source/apidoc/mapchete.log.rst +++ b/doc/source/apidoc/mapchete.log.rst @@ -2,6 +2,6 @@ mapchete.log module =================== .. automodule:: mapchete.log - :members: - :undoc-members: - :show-inheritance: + :members: + :undoc-members: + :show-inheritance: diff --git a/doc/source/apidoc/mapchete.processes.contours.rst b/doc/source/apidoc/mapchete.processes.contours.rst index 9d689fb6..cf2df7d8 100644 --- a/doc/source/apidoc/mapchete.processes.contours.rst +++ b/doc/source/apidoc/mapchete.processes.contours.rst @@ -2,6 +2,6 @@ mapchete.processes.contours module ================================== .. automodule:: mapchete.processes.contours - :members: - :undoc-members: - :show-inheritance: + :members: + :undoc-members: + :show-inheritance: diff --git a/doc/source/apidoc/mapchete.processes.convert.rst b/doc/source/apidoc/mapchete.processes.convert.rst index a8a5b70c..e30a25d4 100644 --- a/doc/source/apidoc/mapchete.processes.convert.rst +++ b/doc/source/apidoc/mapchete.processes.convert.rst @@ -2,6 +2,6 @@ mapchete.processes.convert module ================================= .. automodule:: mapchete.processes.convert - :members: - :undoc-members: - :show-inheritance: + :members: + :undoc-members: + :show-inheritance: diff --git a/doc/source/apidoc/mapchete.processes.examples.example_process.rst b/doc/source/apidoc/mapchete.processes.examples.example_process.rst index d0de5253..cc2ef372 100644 --- a/doc/source/apidoc/mapchete.processes.examples.example_process.rst +++ b/doc/source/apidoc/mapchete.processes.examples.example_process.rst @@ -2,6 +2,6 @@ mapchete.processes.examples.example\_process module =================================================== .. automodule:: mapchete.processes.examples.example_process - :members: - :undoc-members: - :show-inheritance: + :members: + :undoc-members: + :show-inheritance: diff --git a/doc/source/apidoc/mapchete.processes.examples.rst b/doc/source/apidoc/mapchete.processes.examples.rst index 607c9815..39860691 100644 --- a/doc/source/apidoc/mapchete.processes.examples.rst +++ b/doc/source/apidoc/mapchete.processes.examples.rst @@ -5,6 +5,7 @@ Submodules ---------- .. toctree:: + :maxdepth: 4 mapchete.processes.examples.example_process @@ -12,6 +13,6 @@ Module contents --------------- .. automodule:: mapchete.processes.examples - :members: - :undoc-members: - :show-inheritance: + :members: + :undoc-members: + :show-inheritance: diff --git a/doc/source/apidoc/mapchete.processes.hillshade.rst b/doc/source/apidoc/mapchete.processes.hillshade.rst index e330936e..8049c7a9 100644 --- a/doc/source/apidoc/mapchete.processes.hillshade.rst +++ b/doc/source/apidoc/mapchete.processes.hillshade.rst @@ -2,6 +2,6 @@ mapchete.processes.hillshade module =================================== .. automodule:: mapchete.processes.hillshade - :members: - :undoc-members: - :show-inheritance: + :members: + :undoc-members: + :show-inheritance: diff --git a/doc/source/apidoc/mapchete.processes.rst b/doc/source/apidoc/mapchete.processes.rst index 0b3135d6..887d4ce6 100644 --- a/doc/source/apidoc/mapchete.processes.rst +++ b/doc/source/apidoc/mapchete.processes.rst @@ -5,13 +5,15 @@ Subpackages ----------- .. toctree:: + :maxdepth: 4 - mapchete.processes.examples + mapchete.processes.examples Submodules ---------- .. toctree:: + :maxdepth: 4 mapchete.processes.contours mapchete.processes.convert @@ -21,6 +23,6 @@ Module contents --------------- .. automodule:: mapchete.processes - :members: - :undoc-members: - :show-inheritance: + :members: + :undoc-members: + :show-inheritance: diff --git a/doc/source/apidoc/mapchete.rst b/doc/source/apidoc/mapchete.rst index 062cfe6d..588a578d 100644 --- a/doc/source/apidoc/mapchete.rst +++ b/doc/source/apidoc/mapchete.rst @@ -5,17 +5,20 @@ Subpackages ----------- .. toctree:: + :maxdepth: 4 - mapchete.cli - mapchete.commons - mapchete.formats - mapchete.io - mapchete.processes + mapchete.cli + mapchete.commands + mapchete.commons + mapchete.formats + mapchete.io + mapchete.processes Submodules ---------- .. toctree:: + :maxdepth: 4 mapchete.config mapchete.errors @@ -28,6 +31,6 @@ Module contents --------------- .. automodule:: mapchete - :members: - :undoc-members: - :show-inheritance: + :members: + :undoc-members: + :show-inheritance: diff --git a/doc/source/apidoc/mapchete.tile.rst b/doc/source/apidoc/mapchete.tile.rst index 80b55b94..f58e04bd 100644 --- a/doc/source/apidoc/mapchete.tile.rst +++ b/doc/source/apidoc/mapchete.tile.rst @@ -2,6 +2,6 @@ mapchete.tile module ==================== .. automodule:: mapchete.tile - :members: - :undoc-members: - :show-inheritance: + :members: + :undoc-members: + :show-inheritance: diff --git a/doc/source/apidoc/mapchete.validate.rst b/doc/source/apidoc/mapchete.validate.rst index 1bff8c2a..02673e34 100644 --- a/doc/source/apidoc/mapchete.validate.rst +++ b/doc/source/apidoc/mapchete.validate.rst @@ -2,6 +2,6 @@ mapchete.validate module ======================== .. automodule:: mapchete.validate - :members: - :undoc-members: - :show-inheritance: + :members: + :undoc-members: + :show-inheritance: diff --git a/mapchete/commands/_convert.py b/mapchete/commands/_convert.py index 0bdc7433..4b0d4db2 100644 --- a/mapchete/commands/_convert.py +++ b/mapchete/commands/_convert.py @@ -140,16 +140,16 @@ def convert( Examples -------- - >>> execute("foo") + >>> convert("foo", "bar") - This will run the whole copy process. + This will run the whole conversion process. - >>> for i in execute("foo", as_iterator=True): + >>> for i in convert("foo", "bar", as_iterator=True): >>> print(i) This will return a generator where through iteration, tiles are copied. - >>> list(tqdm.tqdm(index("foo", as_iterator=True))) + >>> list(tqdm.tqdm(convert("foo", "bar", as_iterator=True))) Usage within a process bar. """ diff --git a/mapchete/commands/_execute.py b/mapchete/commands/_execute.py index 6b0324d8..f95fb7ad 100755 --- a/mapchete/commands/_execute.py +++ b/mapchete/commands/_execute.py @@ -83,14 +83,14 @@ def execute( -------- >>> execute("foo") - This will run the whole copy process. + This will run the whole execute process. >>> for i in execute("foo", as_iterator=True): >>> print(i) This will return a generator where through iteration, tiles are copied. - >>> list(tqdm.tqdm(index("foo", as_iterator=True))) + >>> list(tqdm.tqdm(execute("foo", as_iterator=True))) Usage within a process bar. """ diff --git a/mapchete/commands/_index.py b/mapchete/commands/_index.py index dfdec483..5ed020d5 100644 --- a/mapchete/commands/_index.py +++ b/mapchete/commands/_index.py @@ -95,7 +95,7 @@ def index( -------- >>> index("foo", vrt=True, zoom=5) - This will run the whole copy process. + This will run the whole index process. >>> for i in index("foo", vrt=True, zoom=5, as_iterator=True): >>> print(i) From 6ba7e72eec0eff76b513b31d7fd6ed3663f9391f Mon Sep 17 00:00:00 2001 From: Joachim Ungar Date: Mon, 19 Jul 2021 12:46:04 +0200 Subject: [PATCH 7/9] add tests --- mapchete/cli/utils.py | 23 -------------- mapchete/commands/_convert.py | 5 +--- mapchete/commands/_execute.py | 2 +- mapchete/commands/_job.py | 12 ++++++-- test/conftest.py | 3 ++ test/test_cli.py | 23 ++++++++++++++ test/test_commands.py | 56 ++++++++++++++++++++++++++++++++++- 7 files changed, 93 insertions(+), 31 deletions(-) diff --git a/mapchete/cli/utils.py b/mapchete/cli/utils.py index 97b50470..99d4764e 100644 --- a/mapchete/cli/utils.py +++ b/mapchete/cli/utils.py @@ -24,20 +24,6 @@ ) -# verbose stdout writer # -######################### -def write_verbose_msg(process_info, dst): - tqdm.tqdm.write( - "Tile %s: %s, %s" - % ( - tuple(process_info.tile.id), - process_info.process_msg, - process_info.write_msg, - ), - file=dst, - ) - - # click callbacks # ################### def _validate_zoom(ctx, param, zoom): @@ -68,14 +54,6 @@ def _validate_mapchete_files(ctx, param, mapchete_files): return mapchete_files -def _validate_inputs(ctx, param, inputs): - if len(inputs) == 0: - raise click.MissingParameter( - "at least one mapchete file or path to Tile Directory required" - ) - return inputs - - def _set_debug_log_level(ctx, param, debug): if debug: set_log_level(logging.DEBUG) @@ -149,7 +127,6 @@ def _cb_key_val(ctx, param, value): arg_input_raster = click.argument("input_raster", type=click.Path(exists=True)) arg_out_dir = click.argument("output_dir", type=click.Path()) arg_input = click.argument("input_", metavar="INPUT", type=click.STRING) -arg_inputs = click.argument("inputs", nargs=-1, callback=_validate_inputs) arg_output = click.argument("output", type=click.STRING) arg_src_tiledir = click.argument("src_tiledir", type=click.STRING) arg_dst_tiledir = click.argument("dst_tiledir", type=click.STRING) diff --git a/mapchete/commands/_convert.py b/mapchete/commands/_convert.py index 4b0d4db2..00afe401 100644 --- a/mapchete/commands/_convert.py +++ b/mapchete/commands/_convert.py @@ -263,10 +263,7 @@ def convert( msg_callback( "Process area is empty: clip bounds don't intersect with input bounds." ) - - def _empty_gen(): - raise StopIteration() - + # this returns a Job with an empty iterator return Job(iter, [], as_iterator=as_iterator, total=0) # add process bounds and output type mapchete_config.update( diff --git a/mapchete/commands/_execute.py b/mapchete/commands/_execute.py index f95fb7ad..1a6a56c1 100755 --- a/mapchete/commands/_execute.py +++ b/mapchete/commands/_execute.py @@ -139,7 +139,7 @@ def execute( total=1 if tile else tiles_count, ) # explicitly exit the mp object on failure - except Exception: + except Exception: # pragma: no cover mp.__exit__(None, None, None) diff --git a/mapchete/commands/_job.py b/mapchete/commands/_job.py index 21f8251a..e5650675 100644 --- a/mapchete/commands/_job.py +++ b/mapchete/commands/_job.py @@ -10,15 +10,17 @@ def __init__( *fargs: dict, as_iterator: bool = False, total: int = None, - **fkwargs: dict + **fkwargs: dict, ): self.func = func self.fargs = fargs self.fkwargs = fkwargs self._total = total self._as_iterator = as_iterator + self._finished = False if not as_iterator: list(self.func(*self.fargs, **self.fkwargs)) + self._finished = True def __len__(self): return self._total @@ -26,7 +28,13 @@ def __len__(self): def __iter__(self): if not self._as_iterator: # pragma: no cover raise TypeError("initialize with 'as_iterator=True'") - return self.func(*self.fargs, **self.fkwargs) + yield from self.func(*self.fargs, **self.fkwargs) + self._finished = True + + def __repr__(self): + return ( + f"" + ) def empty_callback(*args, **kwargs): diff --git a/test/conftest.py b/test/conftest.py index 80860dfd..8183e29e 100644 --- a/test/conftest.py +++ b/test/conftest.py @@ -6,6 +6,7 @@ import pytest from shapely import wkt import shutil +from tempfile import TemporaryDirectory import uuid import yaml @@ -47,6 +48,8 @@ def mp_tmpdir(): os.makedirs(TEMP_DIR) yield TEMP_DIR shutil.rmtree(TEMP_DIR, ignore_errors=True) + # with TemporaryDirectory() as t: + # yield t # temporary directory for I/O tests diff --git a/test/test_cli.py b/test/test_cli.py index aa6e7d40..25163008 100644 --- a/test/test_cli.py +++ b/test/test_cli.py @@ -711,6 +711,29 @@ def test_convert_geobuf_multipolygon(landpoly, mp_tmpdir): assert multipolygons +def test_convert_vrt(cleantopo_br_tif, mp_tmpdir): + """Automatic geodetic tile pyramid creation of raster files.""" + run_cli( + [ + "convert", + cleantopo_br_tif, + mp_tmpdir, + "--output-pyramid", + "geodetic", + "--vrt", + "--zoom", + "1,4", + ] + ) + for zoom in [4, 3, 2, 1]: + out_file = os.path.join(*[mp_tmpdir, str(zoom) + ".vrt"]) + with rasterio.open(out_file, "r") as src: + assert src.meta["driver"] == "VRT" + assert src.meta["dtype"] == "uint16" + data = src.read(masked=True) + assert data.mask.any() + + def test_convert_errors(s2_band_jp2, mp_tmpdir, s2_band, cleantopo_br, landpoly): # output format required run_cli( diff --git a/test/test_commands.py b/test/test_commands.py index fe726564..34f407fd 100644 --- a/test/test_commands.py +++ b/test/test_commands.py @@ -1,7 +1,9 @@ import os +import rasterio +from tilematrix import TilePyramid import mapchete -from mapchete.commands import cp, rm +from mapchete.commands import convert, cp, execute, index, rm SCRIPTDIR = os.path.dirname(os.path.realpath(__file__)) @@ -65,3 +67,55 @@ def test_rm(mp_tmpdir, cleantopo_br): # remove tiles but this time they should already have been removed tiles = rm(out_path, zoom=5) assert len(tiles) == 0 + + +def test_execute(mp_tmpdir, cleantopo_br, cleantopo_br_tif): + zoom = 5 + config = cleantopo_br.dict + config["pyramid"].update(metatiling=1) + tp = TilePyramid("geodetic") + tiles = list(tp.tiles_from_bounds(rasterio.open(cleantopo_br_tif).bounds, zoom)) + job = execute(config, zoom=zoom) + assert len(tiles) == len(job) + with mapchete.open(config) as mp: + for t in tiles: + with rasterio.open(mp.config.output.get_path(t)) as src: + assert not src.read(masked=True).mask.all() + + +def test_execute_tile(mp_tmpdir, cleantopo_br): + tile = (5, 30, 63) + + config = cleantopo_br.dict + config["pyramid"].update(metatiling=1) + job = execute(config, tile=tile) + + assert len(job) == 1 + + with mapchete.open(config) as mp: + with rasterio.open( + mp.config.output.get_path(mp.config.output_pyramid.tile(*tile)) + ) as src: + assert not src.read(masked=True).mask.all() + + +def test_execute_vrt(mp_tmpdir, cleantopo_br): + """Using debug output.""" + execute(cleantopo_br.path, zoom=5, vrt=True) + with mapchete.open(cleantopo_br.dict) as mp: + vrt_path = os.path.join(mp.config.output.path, "5.vrt") + with rasterio.open(vrt_path) as src: + assert src.read().any() + + # run again, this time with custom output directory + execute(cleantopo_br.path, zoom=5, vrt=True, idx_out_dir=mp_tmpdir) + with mapchete.open(cleantopo_br.dict) as mp: + vrt_path = os.path.join(mp_tmpdir, "5.vrt") + with rasterio.open(vrt_path) as src: + assert src.read().any() + + # run with single tile + execute(cleantopo_br.path, tile=(5, 3, 7), vrt=True) + + # no new entries + execute(cleantopo_br.path, tile=(5, 0, 0), vrt=True) From 832ae3a518c39ce8b471072bec5c465a60bc2b81 Mon Sep 17 00:00:00 2001 From: Joachim Ungar Date: Mon, 19 Jul 2021 15:16:05 +0200 Subject: [PATCH 8/9] fix VRT creation --- mapchete/cli/default/rm.py | 2 +- mapchete/commands/_convert.py | 8 ------- mapchete/commands/_execute.py | 6 ------ mapchete/commands/_index.py | 3 ++- mapchete/commands/_job.py | 2 +- mapchete/index.py | 40 +++++++++++++++++++---------------- mapchete/io/__init__.py | 2 -- mapchete/io/_path.py | 25 ---------------------- test/conftest.py | 3 --- test/test_commands.py | 22 ------------------- 10 files changed, 26 insertions(+), 87 deletions(-) diff --git a/mapchete/cli/default/rm.py b/mapchete/cli/default/rm.py index 9d3ef37f..5e02a927 100644 --- a/mapchete/cli/default/rm.py +++ b/mapchete/cli/default/rm.py @@ -46,5 +46,5 @@ def rm( disable=debug or no_pbar, ) ) - else: + else: # pragma: no cover tqdm.tqdm.write("No tiles found to delete.") diff --git a/mapchete/commands/_convert.py b/mapchete/commands/_convert.py index 00afe401..5c26a011 100644 --- a/mapchete/commands/_convert.py +++ b/mapchete/commands/_convert.py @@ -55,8 +55,6 @@ def convert( overviews: bool = False, overviews_resampling_method: str = "cubic_spline", cog: bool = False, - vrt: bool = False, - idx_out_dir=None, msg_callback: Callable = None, as_iterator: bool = False, ) -> Job: @@ -125,10 +123,6 @@ def convert( Resampling method used for overviews. (default: cubic_spline) cog : bool Write a valid COG. This will automatically generate verviews. (GTiff only) - vrt : bool - Activate VRT index creation. - idx_out_dir : str - Alternative output dir for index. Defaults to TileDirectory path. msg_callback : Callable Optional callback function for process messages. as_iterator : bool @@ -284,8 +278,6 @@ def convert( area=area, area_crs=area_crs, multi=multi or cpu_count(), - vrt=vrt, - idx_out_dir=idx_out_dir, as_iterator=as_iterator, msg_callback=msg_callback, ) diff --git a/mapchete/commands/_execute.py b/mapchete/commands/_execute.py index 1a6a56c1..7588792f 100755 --- a/mapchete/commands/_execute.py +++ b/mapchete/commands/_execute.py @@ -26,8 +26,6 @@ def execute( multi: int = None, max_chunksize: int = None, multiprocessing_start_method: str = None, - vrt: bool = False, - idx_out_dir: str = None, msg_callback: Callable = None, as_iterator: bool = False, ) -> Job: @@ -66,10 +64,6 @@ def execute( multiprocessing_start_method : str Method used by multiprocessing module to start child workers. Availability of methods depends on OS. - vrt : bool - Activate VRT index creation. - idx_out_dir : str - Alternative output dir for index. Defaults to TileDirectory path. msg_callback : Callable Optional callback function for process messages. as_iterator : bool diff --git a/mapchete/commands/_index.py b/mapchete/commands/_index.py index 5ed020d5..2b284312 100644 --- a/mapchete/commands/_index.py +++ b/mapchete/commands/_index.py @@ -129,10 +129,11 @@ def index( area=area, area_crs=area_crs, ) as mp: + return Job( zoom_index_gen, mp=mp, - zoom=mp.config.init_zoom_levels, + zoom=None if tile else mp.config.init_zoom_levels, tile=tile, out_dir=idx_out_dir if idx_out_dir else mp.config.output.path, geojson=geojson, diff --git a/mapchete/commands/_job.py b/mapchete/commands/_job.py index e5650675..3e67bf9c 100644 --- a/mapchete/commands/_job.py +++ b/mapchete/commands/_job.py @@ -31,7 +31,7 @@ def __iter__(self): yield from self.func(*self.fargs, **self.fkwargs) self._finished = True - def __repr__(self): + def __repr__(self): # pragma: no cover return ( f"" ) diff --git a/mapchete/index.py b/mapchete/index.py index f4e88465..cfd822fd 100644 --- a/mapchete/index.py +++ b/mapchete/index.py @@ -92,6 +92,9 @@ def zoom_index_gen( use GDAL compatible remote paths, i.e. add "/vsicurl/" before path (default: True) """ + if tile and zoom: # pragma: no cover + raise ValueError("tile and zoom cannot be used at the same time") + zoom = tile[0] if tile else zoom for zoom in get_zoom_levels(process_zoom_levels=zoom): with ExitStack() as es: # get index writers for all enabled formats @@ -150,6 +153,7 @@ def zoom_index_gen( # all output tiles for given process area logger.debug("determine affected output tiles") + logger.debug(f"HANSE {zoom} {tile}") if tile: output_tiles = set( mp.config.output_pyramid.intersecting( @@ -159,26 +163,26 @@ def zoom_index_gen( else: output_tiles = set( [ - tile - for tile in mp.config.output_pyramid.tiles_from_geom( + t + for t in mp.config.output_pyramid.tiles_from_geom( mp.config.area_at_zoom(zoom), zoom ) # this is required to omit tiles touching the config area - if tile.bbox.intersection(mp.config.area_at_zoom(zoom)).area + if t.bbox.intersection(mp.config.area_at_zoom(zoom)).area ] ) - + logger.debug(f"HERBERT {zoom} {output_tiles}") # check which tiles exist in any index logger.debug("check which tiles exist in index(es)") existing_in_any_index = set( - tile - for tile in output_tiles + t + for t in output_tiles if any( [ i.entry_exists( - tile=tile, + tile=t, path=_tile_path( - orig_path=mp.config.output.get_path(tile), + orig_path=mp.config.output.get_path(t), basepath=basepath, for_gdal=for_gdal, ), @@ -194,46 +198,46 @@ def zoom_index_gen( ) ) # tiles which do not exist in any index - for tile, output_exists in tiles_exist( + for t, output_exists in tiles_exist( mp.config, output_tiles=output_tiles.difference(existing_in_any_index) ): tile_path = _tile_path( - orig_path=mp.config.output.get_path(tile), + orig_path=mp.config.output.get_path(t), basepath=basepath, for_gdal=for_gdal, ) indexes = [ i for i in index_writers - if not i.entry_exists(tile=tile, path=tile_path) + if not i.entry_exists(tile=t, path=tile_path) ] if indexes and output_exists: logger.debug("%s exists", tile_path) logger.debug("write to %s indexes" % len(indexes)) for index in indexes: - index.write(tile, tile_path) + index.write(t, tile_path) # yield tile for progress information - yield tile + yield t # tiles which exist in at least one index - for tile in existing_in_any_index: + for t in existing_in_any_index: tile_path = _tile_path( - orig_path=mp.config.output.get_path(tile), + orig_path=mp.config.output.get_path(t), basepath=basepath, for_gdal=for_gdal, ) indexes = [ i for i in index_writers - if not i.entry_exists(tile=tile, path=tile_path) + if not i.entry_exists(tile=t, path=tile_path) ] if indexes: logger.debug("%s exists", tile_path) logger.debug("write to %s indexes" % len(indexes)) for index in indexes: - index.write(tile, tile_path) + index.write(t, tile_path) # yield tile for progress information - yield tile + yield t def _index_file_path(out_dir, zoom, ext): diff --git a/mapchete/io/__init__.py b/mapchete/io/__init__.py index 00b66cdf..8442ad35 100644 --- a/mapchete/io/__init__.py +++ b/mapchete/io/__init__.py @@ -16,7 +16,6 @@ fs_from_path, path_is_remote, path_exists, - rm, tiles_exist, absolute_path, relative_path, @@ -35,7 +34,6 @@ "tiles_exist", "absolute_path", "relative_path", - "rm", "makedirs", "write_json", "read_json", diff --git a/mapchete/io/_path.py b/mapchete/io/_path.py index 1730e803..96053dea 100644 --- a/mapchete/io/_path.py +++ b/mapchete/io/_path.py @@ -100,31 +100,6 @@ def makedirs(path): pass -def rm(paths, fs=None, recursive=False): - """ - Remove one or multiple paths from file system. - - Note: all paths have to be from the same file system! - - Parameters - ---------- - paths : str or list - fs : fsspec.FileSystem - """ - paths = [paths] if isinstance(paths, str) else paths - fs = fs or fs_from_path(paths[0]) - logger.debug(f"got {len(paths)} path(s) on {fs}") - - # s3fs enables multiple paths as input, so let's use this: - if "s3" in fs.protocol: # pragma: no cover - fs.rm(paths, recursive=recursive) - - # otherwise, just iterate through the paths - else: - for path in paths: - fs.rm(path, recursive=recursive) - - def tiles_exist(config=None, output_tiles=None, process_tiles=None, multi=None): """ Yield tiles and whether their output already exists or not. diff --git a/test/conftest.py b/test/conftest.py index 8183e29e..80860dfd 100644 --- a/test/conftest.py +++ b/test/conftest.py @@ -6,7 +6,6 @@ import pytest from shapely import wkt import shutil -from tempfile import TemporaryDirectory import uuid import yaml @@ -48,8 +47,6 @@ def mp_tmpdir(): os.makedirs(TEMP_DIR) yield TEMP_DIR shutil.rmtree(TEMP_DIR, ignore_errors=True) - # with TemporaryDirectory() as t: - # yield t # temporary directory for I/O tests diff --git a/test/test_commands.py b/test/test_commands.py index 34f407fd..88929e6c 100644 --- a/test/test_commands.py +++ b/test/test_commands.py @@ -97,25 +97,3 @@ def test_execute_tile(mp_tmpdir, cleantopo_br): mp.config.output.get_path(mp.config.output_pyramid.tile(*tile)) ) as src: assert not src.read(masked=True).mask.all() - - -def test_execute_vrt(mp_tmpdir, cleantopo_br): - """Using debug output.""" - execute(cleantopo_br.path, zoom=5, vrt=True) - with mapchete.open(cleantopo_br.dict) as mp: - vrt_path = os.path.join(mp.config.output.path, "5.vrt") - with rasterio.open(vrt_path) as src: - assert src.read().any() - - # run again, this time with custom output directory - execute(cleantopo_br.path, zoom=5, vrt=True, idx_out_dir=mp_tmpdir) - with mapchete.open(cleantopo_br.dict) as mp: - vrt_path = os.path.join(mp_tmpdir, "5.vrt") - with rasterio.open(vrt_path) as src: - assert src.read().any() - - # run with single tile - execute(cleantopo_br.path, tile=(5, 3, 7), vrt=True) - - # no new entries - execute(cleantopo_br.path, tile=(5, 0, 0), vrt=True) From 71b2bcdc6b63608f6e295ead3023aa09f299b10d Mon Sep 17 00:00:00 2001 From: Joachim Ungar Date: Mon, 19 Jul 2021 16:47:06 +0200 Subject: [PATCH 9/9] add more tests for command package; fix minor commands issues --- mapchete/commands/_convert.py | 2 + mapchete/commands/_execute.py | 1 + mapchete/commands/_index.py | 2 +- test/conftest.py | 6 + test/test_commands.py | 543 ++++++++++++++++++++++++++++++++++ 5 files changed, 553 insertions(+), 1 deletion(-) diff --git a/mapchete/commands/_convert.py b/mapchete/commands/_convert.py index 5c26a011..cf533053 100644 --- a/mapchete/commands/_convert.py +++ b/mapchete/commands/_convert.py @@ -148,6 +148,8 @@ def convert( Usage within a process bar. """ msg_callback = msg_callback or empty_callback + creation_options = creation_options or {} + bidx = [bidx] if isinstance(bidx, int) else bidx try: input_info = _get_input_info(tiledir) output_info = _get_output_info(output) diff --git a/mapchete/commands/_execute.py b/mapchete/commands/_execute.py index 7588792f..d94eb888 100755 --- a/mapchete/commands/_execute.py +++ b/mapchete/commands/_execute.py @@ -135,6 +135,7 @@ def execute( # explicitly exit the mp object on failure except Exception: # pragma: no cover mp.__exit__(None, None, None) + raise def _msg_wrapper(msg_callback, mp, **kwargs): diff --git a/mapchete/commands/_index.py b/mapchete/commands/_index.py index 2b284312..5b536fb7 100644 --- a/mapchete/commands/_index.py +++ b/mapchete/commands/_index.py @@ -22,7 +22,7 @@ def index( shp: bool = False, vrt: bool = False, txt: bool = False, - fieldname: str = None, + fieldname: str = "location", basepath: str = None, for_gdal: bool = False, zoom: Union[int, List[int]] = None, diff --git a/test/conftest.py b/test/conftest.py index 80860dfd..695224f3 100644 --- a/test/conftest.py +++ b/test/conftest.py @@ -73,6 +73,12 @@ def wkt_geom(): return "Polygon ((2.8125 11.25, 2.8125 14.0625, 0 14.0625, 0 11.25, 2.8125 11.25))" +@pytest.fixture +def wkt_geom_tl(): + """Example WKT geometry.""" + return "Polygon ((-176.04949 85.59671, -174.57652 73.86651, -159.98073 74.58961, -161.74829 83.05249, -176.04949 85.59671))" + + # example files @pytest.fixture def http_raster(): diff --git a/test/test_commands.py b/test/test_commands.py index 88929e6c..45a9f88b 100644 --- a/test/test_commands.py +++ b/test/test_commands.py @@ -1,6 +1,12 @@ +import fiona +import geobuf import os +import pytest import rasterio +from rio_cogeo.cogeo import cog_validate +from shapely.geometry import box, shape from tilematrix import TilePyramid +import warnings import mapchete from mapchete.commands import convert, cp, execute, index, rm @@ -97,3 +103,540 @@ def test_execute_tile(mp_tmpdir, cleantopo_br): mp.config.output.get_path(mp.config.output_pyramid.tile(*tile)) ) as src: assert not src.read(masked=True).mask.all() + + +def test_execute_point(mp_tmpdir, example_mapchete, dummy2_tif): + """Using bounds from WKT.""" + with rasterio.open(dummy2_tif) as src: + g = box(*src.bounds) + job = execute(example_mapchete.path, point=[g.centroid.x, g.centroid.y], zoom=10) + assert len(job) == 1 + + +def test_convert_geodetic(cleantopo_br_tif, mp_tmpdir): + """Automatic geodetic tile pyramid creation of raster files.""" + job = convert(cleantopo_br_tif, mp_tmpdir, output_pyramid="geodetic") + assert len(job) + for zoom, row, col in [(4, 15, 31), (3, 7, 15), (2, 3, 7), (1, 1, 3)]: + out_file = os.path.join(*[mp_tmpdir, str(zoom), str(row), str(col) + ".tif"]) + with rasterio.open(out_file, "r") as src: + assert src.meta["driver"] == "GTiff" + assert src.meta["dtype"] == "uint16" + data = src.read(masked=True) + assert data.mask.any() + + +def test_convert_mercator(cleantopo_br_tif, mp_tmpdir): + """Automatic mercator tile pyramid creation of raster files.""" + job = convert(cleantopo_br_tif, mp_tmpdir, output_pyramid="mercator") + assert len(job) + for zoom, row, col in [(4, 15, 15), (3, 7, 7)]: + out_file = os.path.join(*[mp_tmpdir, str(zoom), str(row), str(col) + ".tif"]) + with rasterio.open(out_file, "r") as src: + assert src.meta["driver"] == "GTiff" + assert src.meta["dtype"] == "uint16" + data = src.read(masked=True) + assert data.mask.any() + + +def test_convert_png(cleantopo_br_tif, mp_tmpdir): + """Automatic PNG tile pyramid creation of raster files.""" + job = convert( + cleantopo_br_tif, mp_tmpdir, output_pyramid="mercator", output_format="PNG" + ) + assert len(job) + for zoom, row, col in [(4, 15, 15), (3, 7, 7)]: + out_file = os.path.join(*[mp_tmpdir, str(zoom), str(row), str(col) + ".png"]) + with warnings.catch_warnings(): + warnings.simplefilter("ignore") + with rasterio.open(out_file, "r") as src: + assert src.meta["driver"] == "PNG" + assert src.meta["dtype"] == "uint8" + data = src.read(masked=True) + assert data.mask.any() + + +def test_convert_bidx(cleantopo_br_tif, mp_tmpdir): + """Automatic geodetic tile pyramid creation of raster files.""" + single_gtiff = os.path.join(mp_tmpdir, "single_out_bidx.tif") + job = convert( + cleantopo_br_tif, single_gtiff, output_pyramid="geodetic", zoom=3, bidx=1 + ) + assert len(job) + with rasterio.open(single_gtiff, "r") as src: + assert src.meta["driver"] == "GTiff" + assert src.meta["dtype"] == "uint16" + data = src.read(masked=True) + assert data.mask.any() + assert not src.overviews(1) + + +def test_convert_single_gtiff(cleantopo_br_tif, mp_tmpdir): + """Automatic geodetic tile pyramid creation of raster files.""" + single_gtiff = os.path.join(mp_tmpdir, "single_out.tif") + job = convert(cleantopo_br_tif, single_gtiff, output_pyramid="geodetic", zoom=3) + assert len(job) + with rasterio.open(single_gtiff, "r") as src: + assert src.meta["driver"] == "GTiff" + assert src.meta["dtype"] == "uint16" + data = src.read(masked=True) + assert data.mask.any() + assert not src.overviews(1) + + +def test_convert_single_gtiff_cog(cleantopo_br_tif, mp_tmpdir): + """Automatic geodetic tile pyramid creation of raster files.""" + single_gtiff = os.path.join(mp_tmpdir, "single_out_cog.tif") + job = convert( + cleantopo_br_tif, single_gtiff, output_pyramid="geodetic", zoom=3, cog=True + ) + assert len(job) + with rasterio.open(single_gtiff, "r") as src: + assert src.meta["driver"] == "GTiff" + assert src.meta["dtype"] == "uint16" + data = src.read(masked=True) + assert data.mask.any() + assert cog_validate(single_gtiff, strict=True) + + +def test_convert_single_gtiff_overviews(cleantopo_br_tif, mp_tmpdir): + """Automatic geodetic tile pyramid creation of raster files.""" + single_gtiff = os.path.join(mp_tmpdir, "single_out.tif") + job = convert( + cleantopo_br_tif, + single_gtiff, + output_pyramid="geodetic", + zoom=7, + overviews=True, + overviews_resampling_method="bilinear", + ) + assert len(job) + with rasterio.open(single_gtiff, "r") as src: + assert src.meta["driver"] == "GTiff" + assert src.meta["dtype"] == "uint16" + data = src.read(masked=True) + assert data.mask.any() + assert src.overviews(1) + + +def test_convert_remote_single_gtiff(http_raster, mp_tmpdir): + """Automatic geodetic tile pyramid creation of raster files.""" + single_gtiff = os.path.join(mp_tmpdir, "single_out.tif") + job = convert(http_raster, single_gtiff, output_pyramid="geodetic", zoom=1) + assert len(job) + with rasterio.open(single_gtiff, "r") as src: + assert src.meta["driver"] == "GTiff" + assert src.meta["dtype"] == "uint16" + data = src.read(masked=True) + assert data.any() + + +def test_convert_dtype(cleantopo_br_tif, mp_tmpdir): + """Automatic tile pyramid creation using dtype scale.""" + job = convert( + cleantopo_br_tif, mp_tmpdir, output_pyramid="mercator", output_dtype="uint8" + ) + assert len(job) + for zoom, row, col in [(4, 15, 15), (3, 7, 7)]: + out_file = os.path.join(*[mp_tmpdir, str(zoom), str(row), str(col) + ".tif"]) + with rasterio.open(out_file, "r") as src: + assert src.meta["driver"] == "GTiff" + assert src.meta["dtype"] == "uint8" + data = src.read(masked=True) + assert data.mask.any() + + +def test_convert_scale_ratio(cleantopo_br_tif, mp_tmpdir): + """Automatic tile pyramid creation cropping data.""" + job = convert( + cleantopo_br_tif, + mp_tmpdir, + output_pyramid="mercator", + output_dtype="uint8", + scale_ratio=0.003, + ) + print(job) + assert len(job) + for zoom, row, col in [(4, 15, 15), (3, 7, 7)]: + out_file = os.path.join(*[mp_tmpdir, str(zoom), str(row), str(col) + ".tif"]) + with rasterio.open(out_file, "r") as src: + assert src.meta["driver"] == "GTiff" + assert src.meta["dtype"] == "uint8" + data = src.read(masked=True) + assert data.mask.any() + assert not data.mask.all() + + +def test_convert_scale_offset(cleantopo_br_tif, mp_tmpdir): + """Automatic tile pyramid creation cropping data.""" + job = convert( + cleantopo_br_tif, + mp_tmpdir, + output_pyramid="mercator", + output_dtype="uint8", + scale_offset=1, + ) + assert len(job) + for zoom, row, col in [(4, 15, 15), (3, 7, 7)]: + out_file = os.path.join(*[mp_tmpdir, str(zoom), str(row), str(col) + ".tif"]) + with rasterio.open(out_file, "r") as src: + assert src.meta["driver"] == "GTiff" + assert src.meta["dtype"] == "uint8" + data = src.read(masked=True) + assert data.mask.any() + assert not data.mask.all() + + +def test_convert_clip(cleantopo_br_tif, mp_tmpdir, landpoly): + """Automatic tile pyramid creation cropping data.""" + job = convert( + cleantopo_br_tif, mp_tmpdir, output_pyramid="geodetic", clip_geometry=landpoly + ) + assert len(job) == 0 + + +def test_convert_zoom(cleantopo_br_tif, mp_tmpdir): + """Automatic tile pyramid creation using a specific zoom.""" + job = convert(cleantopo_br_tif, mp_tmpdir, output_pyramid="mercator", zoom=3) + assert len(job) + for zoom, row, col in [(4, 15, 15), (2, 3, 0)]: + out_file = os.path.join(*[mp_tmpdir, str(zoom), str(row), str(col) + ".tif"]) + assert not os.path.isfile(out_file) + + +def test_convert_zoom_minmax(cleantopo_br_tif, mp_tmpdir): + """Automatic tile pyramid creation using min max zoom.""" + job = convert(cleantopo_br_tif, mp_tmpdir, output_pyramid="mercator", zoom=[3, 4]) + assert len(job) + for zoom, row, col in [(2, 3, 0)]: + out_file = os.path.join(*[mp_tmpdir, str(zoom), str(row), str(col) + ".tif"]) + assert not os.path.isfile(out_file) + + +def test_convert_zoom_maxmin(cleantopo_br_tif, mp_tmpdir): + """Automatic tile pyramid creation using max min zoom.""" + job = convert(cleantopo_br_tif, mp_tmpdir, output_pyramid="mercator", zoom=[4, 3]) + assert len(job) + for zoom, row, col in [(2, 3, 0)]: + out_file = os.path.join(*[mp_tmpdir, str(zoom), str(row), str(col) + ".tif"]) + assert not os.path.isfile(out_file) + + +def test_convert_mapchete(cleantopo_br, mp_tmpdir): + # prepare data + job = execute(cleantopo_br.path, zoom=[1, 4]) + assert len(job) + + job = convert( + cleantopo_br.path, mp_tmpdir, output_pyramid="geodetic", output_metatiling=1 + ) + assert len(job) + for zoom, row, col in [(4, 15, 31), (3, 7, 15), (2, 3, 7), (1, 1, 3)]: + out_file = os.path.join(*[mp_tmpdir, str(zoom), str(row), str(col) + ".tif"]) + with rasterio.open(out_file, "r") as src: + assert src.meta["driver"] == "GTiff" + assert src.meta["dtype"] == "uint16" + data = src.read(masked=True) + assert data.mask.any() + + +def test_convert_tiledir(cleantopo_br, mp_tmpdir): + # prepare data + with mapchete.open(cleantopo_br.path) as mp: + mp.batch_process(zoom=[1, 4]) + job = convert( + os.path.join( + cleantopo_br.dict["config_dir"], cleantopo_br.dict["output"]["path"] + ), + mp_tmpdir, + output_pyramid="geodetic", + output_metatiling=1, + zoom=[1, 4], + ) + assert len(job) + for zoom, row, col in [(4, 15, 31), (3, 7, 15), (2, 3, 7), (1, 1, 3)]: + out_file = os.path.join(*[mp_tmpdir, str(zoom), str(row), str(col) + ".tif"]) + with rasterio.open(out_file, "r") as src: + assert src.meta["driver"] == "GTiff" + assert src.meta["dtype"] == "uint16" + data = src.read(masked=True) + assert data.mask.any() + + +def test_convert_geojson(landpoly, mp_tmpdir): + job = convert(landpoly, mp_tmpdir, output_pyramid="geodetic", zoom=4) + assert len(job) + for (zoom, row, col), control in zip([(4, 0, 7), (4, 1, 7)], [9, 32]): + out_file = os.path.join( + *[mp_tmpdir, str(zoom), str(row), str(col) + ".geojson"] + ) + with fiona.open(out_file, "r") as src: + assert len(src) == control + for f in src: + assert shape(f["geometry"]).is_valid + + +def test_convert_geobuf(landpoly, mp_tmpdir): + # convert to geobuf + geobuf_outdir = os.path.join(mp_tmpdir, "geobuf") + job = convert( + landpoly, + geobuf_outdir, + output_pyramid="geodetic", + zoom=4, + output_format="Geobuf", + ) + assert len(job) + for (zoom, row, col), control in zip([(4, 0, 7), (4, 1, 7)], [9, 32]): + out_file = os.path.join( + *[geobuf_outdir, str(zoom), str(row), str(col) + ".pbf"] + ) + with open(out_file, "rb") as src: + features = geobuf.decode(src.read())["features"] + assert len(features) == control + for f in features: + assert f["geometry"]["type"] == "Polygon" + assert shape(f["geometry"]).area + + +def test_convert_errors(s2_band_jp2, mp_tmpdir, s2_band, cleantopo_br, landpoly): + # output format required + with pytest.raises(ValueError): + convert(s2_band_jp2, mp_tmpdir, output_pyramid="geodetic") + + # output pyramid reqired + with pytest.raises(ValueError): + convert(s2_band, mp_tmpdir) + + # prepare data for tiledir input + with mapchete.open(cleantopo_br.path) as mp: + mp.batch_process(zoom=[1, 4]) + tiledir_path = os.path.join( + cleantopo_br.dict["config_dir"], cleantopo_br.dict["output"]["path"] + ) + + # zoom level required + with pytest.raises(ValueError): + convert(tiledir_path, mp_tmpdir, output_pyramid="geodetic") + + # incompatible formats + with pytest.raises(ValueError): + convert( + tiledir_path, + mp_tmpdir, + output_pyramid="geodetic", + zoom=5, + output_format="GeoJSON", + ) + + # unsupported output format extension + with pytest.raises(ValueError): + convert(s2_band_jp2, "output.jp2", output_pyramid="geodetic", zoom=5) + + # malformed band index + with pytest.raises(ValueError): + convert(s2_band_jp2, "output.tif", bidx="invalid") + + +def test_index_geojson(mp_tmpdir, cleantopo_br): + # execute process at zoom 3 + job = execute(cleantopo_br.path, zoom=3) + assert len(job) + + # generate index for zoom 3 + job = index(cleantopo_br.path, zoom=3, geojson=True) + assert len(job) + + with mapchete.open(cleantopo_br.dict) as mp: + files = os.listdir(mp.config.output.path) + assert len(files) == 3 + assert "3.geojson" in files + with fiona.open(os.path.join(mp.config.output.path, "3.geojson")) as src: + for f in src: + assert "location" in f["properties"] + assert len(list(src)) == 1 + + +def test_index_geojson_fieldname(mp_tmpdir, cleantopo_br): + # execute process at zoom 3 + job = execute(cleantopo_br.path, zoom=3) + assert len(job) + + # index and rename "location" to "new_fieldname" + job = index( + cleantopo_br.path, + zoom=3, + geojson=True, + fieldname="new_fieldname", + ) + assert len(job) + with mapchete.open(cleantopo_br.dict) as mp: + files = os.listdir(mp.config.output.path) + assert "3.geojson" in files + with fiona.open(os.path.join(mp.config.output.path, "3.geojson")) as src: + for f in src: + assert "new_fieldname" in f["properties"] + assert len(list(src)) == 1 + + +def test_index_geojson_basepath(mp_tmpdir, cleantopo_br): + # execute process at zoom 3 + job = execute(cleantopo_br.path, zoom=3) + assert len(job) + + basepath = "http://localhost" + # index and rename "location" to "new_fieldname" + job = index(cleantopo_br.path, zoom=3, geojson=True, basepath=basepath) + assert len(job) + + with mapchete.open(cleantopo_br.dict) as mp: + files = os.listdir(mp.config.output.path) + assert "3.geojson" in files + with fiona.open(os.path.join(mp.config.output.path, "3.geojson")) as src: + for f in src: + assert f["properties"]["location"].startswith(basepath) + assert len(list(src)) == 1 + + +def test_index_geojson_for_gdal(mp_tmpdir, cleantopo_br): + # execute process at zoom 3 + job = execute(cleantopo_br.path, zoom=3) + assert len(job) + + basepath = "http://localhost" + # index and rename "location" to "new_fieldname" + job = index( + cleantopo_br.path, zoom=3, geojson=True, basepath=basepath, for_gdal=True + ) + assert len(job) + + with mapchete.open(cleantopo_br.dict) as mp: + files = os.listdir(mp.config.output.path) + assert "3.geojson" in files + with fiona.open(os.path.join(mp.config.output.path, "3.geojson")) as src: + for f in src: + assert f["properties"]["location"].startswith("/vsicurl/" + basepath) + assert len(list(src)) == 1 + + +def test_index_geojson_tile(mp_tmpdir, cleantopo_tl): + # execute process at zoom 3 + job = execute(cleantopo_tl.path, zoom=3) + assert len(job) + + # generate index + job = index(cleantopo_tl.path, tile=(3, 0, 0), geojson=True) + assert len(job) + + with mapchete.open(cleantopo_tl.dict) as mp: + files = os.listdir(mp.config.output.path) + assert len(files) == 3 + assert "3.geojson" in files + with fiona.open(os.path.join(mp.config.output.path, "3.geojson")) as src: + assert len(list(src)) == 1 + + +def test_index_geojson_wkt_area(mp_tmpdir, cleantopo_tl, wkt_geom_tl): + # execute process at zoom 3 + job = execute(cleantopo_tl.path, area=wkt_geom_tl) + assert len(job) + + # generate index for zoom 3 + job = index(cleantopo_tl.path, geojson=True, area=wkt_geom_tl) + assert len(job) + + with mapchete.open(cleantopo_tl.dict) as mp: + files = os.listdir(mp.config.output.path) + assert len(files) == 13 + assert "3.geojson" in files + + +def test_index_gpkg(mp_tmpdir, cleantopo_br): + # execute process + job = execute(cleantopo_br.path, zoom=5) + assert len(job) + + # generate index + job = index(cleantopo_br.path, zoom=5, gpkg=True) + assert len(job) + with mapchete.open(cleantopo_br.dict) as mp: + files = os.listdir(mp.config.output.path) + assert "5.gpkg" in files + with fiona.open(os.path.join(mp.config.output.path, "5.gpkg")) as src: + for f in src: + assert "location" in f["properties"] + assert len(list(src)) == 1 + + # write again and assert there is no new entry because there is already one + job = index(cleantopo_br.path, zoom=5, gpkg=True) + assert len(job) + with mapchete.open(cleantopo_br.dict) as mp: + files = os.listdir(mp.config.output.path) + assert "5.gpkg" in files + with fiona.open(os.path.join(mp.config.output.path, "5.gpkg")) as src: + for f in src: + assert "location" in f["properties"] + assert len(list(src)) == 1 + + +def test_index_shp(mp_tmpdir, cleantopo_br): + # execute process + job = execute(cleantopo_br.path, zoom=5) + assert len(job) + + # generate index + job = index(cleantopo_br.path, zoom=5, shp=True) + assert len(job) + with mapchete.open(cleantopo_br.dict) as mp: + files = os.listdir(mp.config.output.path) + assert "5.shp" in files + with fiona.open(os.path.join(mp.config.output.path, "5.shp")) as src: + for f in src: + assert "location" in f["properties"] + assert len(list(src)) == 1 + + # write again and assert there is no new entry because there is already one + job = index(cleantopo_br.path, zoom=5, shp=True) + assert len(job) + with mapchete.open(cleantopo_br.dict) as mp: + files = os.listdir(mp.config.output.path) + assert "5.shp" in files + with fiona.open(os.path.join(mp.config.output.path, "5.shp")) as src: + for f in src: + assert "location" in f["properties"] + assert len(list(src)) == 1 + + +def test_index_text(cleantopo_br): + # execute process + job = execute(cleantopo_br.path, zoom=5) + assert len(job) + + # generate index + job = index(cleantopo_br.path, zoom=5, txt=True) + assert len(job) + with mapchete.open(cleantopo_br.dict) as mp: + files = os.listdir(mp.config.output.path) + assert "5.txt" in files + with open(os.path.join(mp.config.output.path, "5.txt")) as src: + lines = list(src) + assert len(lines) == 1 + for l in lines: + assert l.endswith("7.tif\n") + + # write again and assert there is no new entry because there is already one + job = index(cleantopo_br.path, zoom=5, txt=True) + assert len(job) + with mapchete.open(cleantopo_br.dict) as mp: + files = os.listdir(mp.config.output.path) + assert "5.txt" in files + with open(os.path.join(mp.config.output.path, "5.txt")) as src: + lines = list(src) + assert len(lines) == 1 + for l in lines: + assert l.endswith("7.tif\n") + + +def test_index_errors(mp_tmpdir, cleantopo_br): + with pytest.raises(ValueError): + index(cleantopo_br.path, zoom=5)