From 2fdf67c3dd2bda3945f07ce3f3b5f758f995152d Mon Sep 17 00:00:00 2001 From: Joachim Ungar Date: Fri, 29 Oct 2021 11:04:04 +0200 Subject: [PATCH 1/9] add initialize_inputs() --- mapchete/config.py | 114 ++++++++++++++++++++++++-------------------- mapchete/testing.py | 26 ++++++++++ 2 files changed, 88 insertions(+), 52 deletions(-) create mode 100644 mapchete/testing.py diff --git a/mapchete/config.py b/mapchete/config.py index c33957f9..2c24d034 100644 --- a/mapchete/config.py +++ b/mapchete/config.py @@ -478,61 +478,17 @@ def input(self): ] ) - initalized_inputs = OrderedDict() - if self._init_inputs: - for k, v in raw_inputs.items(): - # for files and tile directories - if isinstance(v, str): - logger.debug("load input reader for simple input %s", v) - try: - reader = load_input_reader( - dict( - path=absolute_path(path=v, base_dir=self.config_dir), - pyramid=self.process_pyramid, - pixelbuffer=self.process_pyramid.pixelbuffer, - delimiters=self._delimiters, - ), - readonly=self.mode == "readonly", - ) - except Exception as e: - logger.exception(e) - raise MapcheteDriverError( - "error when loading input %s: %s" % (v, e) - ) - logger.debug("input reader for simple input %s is %s", v, reader) - - # for abstract inputs - elif isinstance(v, dict): - logger.debug("load input reader for abstract input %s", v) - try: - reader = load_input_reader( - dict( - abstract=deepcopy(v), - pyramid=self.process_pyramid, - pixelbuffer=self.process_pyramid.pixelbuffer, - delimiters=self._delimiters, - conf_dir=self.config_dir, - ), - readonly=self.mode == "readonly", - ) - except Exception as e: - logger.exception(e) - raise MapcheteDriverError( - "error when loading input %s: %s" % (v, e) - ) - logger.debug("input reader for abstract input %s is %s", v, reader) - else: - raise MapcheteConfigError("invalid input type %s", type(v)) - # trigger bbox creation - reader.bbox(out_crs=self.process_pyramid.crs) - initalized_inputs[k] = reader + return initialize_inputs( + raw_inputs, + config_dir=self.config_dir, + pyramid=self.process_pyramid, + delimiters=self._delimiters, + readonly=self.mode == "readonly", + ) else: - for k in raw_inputs.keys(): - initalized_inputs[k] = None - - return initalized_inputs + return OrderedDict([(k, None) for k in raw_inputs.keys()]) @cached_property def baselevels(self): @@ -1052,6 +1008,60 @@ def get_process_func(process=None, config_dir=None, run_compile=False): raise MapcheteProcessImportError(e) +def initialize_inputs( + raw_inputs, + config_dir=None, + pyramid=None, + delimiters=None, + readonly=False, +): + initalized_inputs = OrderedDict() + for k, v in raw_inputs.items(): + # for files and tile directories + if isinstance(v, str): + logger.debug("load input reader for simple input %s", v) + try: + reader = load_input_reader( + dict( + path=absolute_path(path=v, base_dir=config_dir), + pyramid=pyramid, + pixelbuffer=pyramid.pixelbuffer, + delimiters=delimiters, + ), + readonly=readonly, + ) + except Exception as e: + logger.exception(e) + raise MapcheteDriverError("error when loading input %s: %s" % (v, e)) + logger.debug("input reader for simple input %s is %s", v, reader) + + # for abstract inputs + elif isinstance(v, dict): + logger.debug("load input reader for abstract input %s", v) + try: + reader = load_input_reader( + dict( + abstract=deepcopy(v), + pyramid=pyramid, + pixelbuffer=pyramid.pixelbuffer, + delimiters=delimiters, + conf_dir=config_dir, + ), + readonly=readonly, + ) + except Exception as e: + logger.exception(e) + raise MapcheteDriverError("error when loading input %s: %s" % (v, e)) + logger.debug("input reader for abstract input %s is %s", v, reader) + else: + raise MapcheteConfigError("invalid input type %s", type(v)) + # trigger bbox creation + reader.bbox(out_crs=pyramid.crs) + initalized_inputs[k] = reader + + return initalized_inputs + + def _load_process_module(process=None, config_dir=None, run_compile=False): tmpfile = None try: diff --git a/mapchete/testing.py b/mapchete/testing.py new file mode 100644 index 00000000..299fa05b --- /dev/null +++ b/mapchete/testing.py @@ -0,0 +1,26 @@ +""" +Useful tools to facilitate testing. +""" +from mapchete import MapcheteProcess +from mapchete.tile import BufferedTilePyramid + + +def parse_inputs(inputs): + 1 / 0 + + +def mp(tile=None, params=None, input=None, output_params=None, config=None): + """ + Return a MapcheteProcess object which can be used in a process function. + + e.g. execute(mp) + """ + tile = tile or BufferedTilePyramid("geodetic").tile(0, 0, 0) + input = input or parse_inputs(input) + return MapcheteProcess( + tile=tile, + params=params, + input=input, + output_params=output_params, + config=config, + ) From b4c53898731d1746ff2748dd613d522500fe2d5b Mon Sep 17 00:00:00 2001 From: Joachim Ungar Date: Tue, 2 Nov 2021 19:37:14 +0100 Subject: [PATCH 2/9] add improved test fixture for processes --- mapchete/config.py | 23 +-- mapchete/testing.py | 123 +++++++++++++-- test/conftest.py | 241 ++++++++++++++++++------------ test/test_commands.py | 36 ++--- test/test_config.py | 6 - test/test_processes.py | 332 +++++++++++++++-------------------------- 6 files changed, 397 insertions(+), 364 deletions(-) diff --git a/mapchete/config.py b/mapchete/config.py index 2c24d034..225ece12 100644 --- a/mapchete/config.py +++ b/mapchete/config.py @@ -560,16 +560,9 @@ def get_process_func_params(self, zoom): def get_inputs_for_tile(self, tile): """Get and open all inputs for given tile.""" - def _open_inputs(i): - for k, v in i.items(): - if v is None: - continue - elif isinstance(v, dict): - yield (k, list(_open_inputs(v))) - else: - yield (k, v.open(tile)) - - return OrderedDict(list(_open_inputs(self.params_at_zoom(tile.zoom)["input"]))) + return OrderedDict( + list(open_inputs(self.params_at_zoom(tile.zoom)["input"], tile)) + ) def params_at_zoom(self, zoom): """ @@ -1105,6 +1098,16 @@ def _load_process_module(process=None, config_dir=None, run_compile=False): return module +def open_inputs(inputs, tile): + for k, v in inputs.items(): + if v is None: + continue + elif isinstance(v, dict): + yield (k, list(open_inputs(v, tile))) + else: + yield (k, v.open(tile)) + + def _config_to_dict(input_config): if isinstance(input_config, dict): if "config_dir" not in input_config: diff --git a/mapchete/testing.py b/mapchete/testing.py index 299fa05b..dbf07640 100644 --- a/mapchete/testing.py +++ b/mapchete/testing.py @@ -1,26 +1,117 @@ """ Useful tools to facilitate testing. """ -from mapchete import MapcheteProcess -from mapchete.tile import BufferedTilePyramid +from collections import OrderedDict +import logging +import os +import oyaml as yaml +from shapely.ops import unary_union +import uuid +import mapchete +from mapchete.config import initialize_inputs, open_inputs +from mapchete.io import fs_from_path +from mapchete.tile import BufferedTile, BufferedTilePyramid -def parse_inputs(inputs): - 1 / 0 +logger = logging.getLogger(__name__) -def mp(tile=None, params=None, input=None, output_params=None, config=None): - """ - Return a MapcheteProcess object which can be used in a process function. - e.g. execute(mp) +# helper functions +def dict_from_mapchete(path): + """ + Read mapchete configuration from file and return as dictionary. """ - tile = tile or BufferedTilePyramid("geodetic").tile(0, 0, 0) - input = input or parse_inputs(input) - return MapcheteProcess( - tile=tile, - params=params, - input=input, - output_params=output_params, - config=config, + with open(path) as src: + return dict(yaml.safe_load(src.read()), config_dir=os.path.dirname(path)) + + +class ProcessFixture: + def __init__(self, path=None, output_tempdir=None): + self.path = path + self.dict = None + if output_tempdir: + self._output_tempdir = os.path.join(output_tempdir, uuid.uuid4().hex) + else: + self._output_tempdir = None + self._out_fs = None + self._mp = None + + def __enter__(self, *args): + self.dict = dict_from_mapchete(self.path) + if self._output_tempdir: + # set output directory + self.dict["output"]["path"] = self._output_tempdir + + self._mp = mapchete.open(self.dict) + return self + + def __exit__(self, *args): + # properly close mapchete + if self._mp: + self._mp.__exit__(*args) + self.clear_output() + + def clear_output(self): + # delete written output if any + out_dir = self.dict["output"]["path"] + if self._output_tempdir: + try: + fs_from_path(out_dir).rm(out_dir, recursive=True) + except FileNotFoundError: + pass + + def process_mp(self, tile=None, tile_zoom=None): + """ + Return MapcheteProcess object used to test processes. + """ + if tile: + tile = self._mp.config.process_pyramid.tile(*tile) + else: + # just use first process tile from lowest zoom level + zoom = tile_zoom or max(self._mp.config.zoom_levels) + tile = next(self._mp.get_process_tiles(zoom)) + return mapchete.MapcheteProcess( + tile=tile, + params=self._mp.config.params_at_zoom(tile.zoom), + input=self._mp.config.get_inputs_for_tile(tile), + ) + + def mp(self): + """ + Return Mapchete object. + """ + return self._mp + + +def get_process_mp(tile=None, zoom=0, input=None, params=None, metatiling=1, **kwargs): + if isinstance(tile, BufferedTile): + pyramid = BufferedTile.tile_pyramid + elif isinstance(tile, tuple): + pyramid = BufferedTilePyramid("geodetic", metatiling=metatiling) + tile = pyramid.tile(*tile) + else: + pyramid = BufferedTilePyramid("geodetic", metatiling=metatiling) + initialized_inputs = initialize_inputs( + input, + config_dir=None, + pyramid=pyramid, + delimiters=None, + readonly=False, + ) + if tile is None: + if zoom is None: # pragma: no cover + raise ValueError("either tile or tile_zoom have to be provided") + tile = next( + pyramid.tiles_from_geom( + unary_union( + [v.bbox(out_crs=pyramid.crs) for v in initialized_inputs.values()] + ), + zoom, + ) + ) + logger.debug(f"tile is {tile}") + inputs = OrderedDict(open_inputs(initialized_inputs, tile)) + return mapchete.MapcheteProcess( + tile=tile, input=inputs, params=params or {}, **kwargs ) diff --git a/test/conftest.py b/test/conftest.py index 5f351412..a142cdd8 100644 --- a/test/conftest.py +++ b/test/conftest.py @@ -10,6 +10,9 @@ import yaml from mapchete.cli.default.serve import create_app +from mapchete.io import fs_from_path +from mapchete.testing import ProcessFixture, dict_from_mapchete + SCRIPT_DIR = os.path.dirname(os.path.realpath(__file__)) TESTDATA_DIR = os.path.join(SCRIPT_DIR, "testdata") @@ -17,9 +20,6 @@ S3_TEMP_DIR = "s3://mapchete-test/tmp/" + uuid.uuid4().hex -ExampleConfig = namedtuple("ExampleConfig", ("path", "dict")) - - # flask test app for mapchete serve @pytest.fixture def app(dem_to_hillshade, cleantopo_br, geobuf, geojson, mp_tmpdir): @@ -53,14 +53,13 @@ def mp_tmpdir(): @pytest.fixture def mp_s3_tmpdir(): """Setup and teardown temporary directory.""" + fs = fs_from_path(S3_TEMP_DIR) def _cleanup(): - for obj in ( - boto3.resource("s3") - .Bucket(S3_TEMP_DIR.split("/")[2]) - .objects.filter(Prefix="/".join(S3_TEMP_DIR.split("/")[-2:])) - ): - obj.delete() + try: + fs.rm(S3_TEMP_DIR, recursive=True) + except FileNotFoundError: + pass _cleanup() yield S3_TEMP_DIR @@ -280,257 +279,305 @@ def old_style_process_py(): @pytest.fixture def custom_grid(): """Fixture for custom_grid.mapchete.""" - path = os.path.join(TESTDATA_DIR, "custom_grid.mapchete") - return ExampleConfig(path=path, dict=_dict_from_mapchete(path)) + with ProcessFixture( + os.path.join(TESTDATA_DIR, "custom_grid.mapchete"), + ) as example: + yield example @pytest.fixture def deprecated_params(): """Fixture for deprecated_params.mapchete.""" - path = os.path.join(TESTDATA_DIR, "deprecated_params.mapchete") - return ExampleConfig(path=path, dict=_dict_from_mapchete(path)) - - -@pytest.fixture -def abstract_input(): - """Fixture for abstract_input.mapchete.""" - path = os.path.join(TESTDATA_DIR, "abstract_input.mapchete") - return ExampleConfig(path=path, dict=_dict_from_mapchete(path)) + with ProcessFixture( + os.path.join(TESTDATA_DIR, "deprecated_params.mapchete"), + ) as example: + yield example @pytest.fixture def files_zooms(): """Fixture for files_zooms.mapchete.""" - path = os.path.join(TESTDATA_DIR, "files_zooms.mapchete") - return ExampleConfig(path=path, dict=_dict_from_mapchete(path)) + with ProcessFixture( + os.path.join(TESTDATA_DIR, "files_zooms.mapchete"), + ) as example: + yield example @pytest.fixture def file_groups(): """Fixture for file_groups.mapchete.""" - path = os.path.join(TESTDATA_DIR, "file_groups.mapchete") - return ExampleConfig(path=path, dict=_dict_from_mapchete(path)) + with ProcessFixture( + os.path.join(TESTDATA_DIR, "file_groups.mapchete"), + ) as example: + yield example @pytest.fixture def baselevels(): """Fixture for baselevels.mapchete.""" - path = os.path.join(TESTDATA_DIR, "baselevels.mapchete") - return ExampleConfig(path=path, dict=_dict_from_mapchete(path)) + with ProcessFixture( + os.path.join(TESTDATA_DIR, "baselevels.mapchete"), + ) as example: + yield example @pytest.fixture def baselevels_output_buffer(): """Fixture for baselevels_output_buffer.mapchete.""" - path = os.path.join(TESTDATA_DIR, "baselevels_output_buffer.mapchete") - return ExampleConfig(path=path, dict=_dict_from_mapchete(path)) + with ProcessFixture( + os.path.join(TESTDATA_DIR, "baselevels_output_buffer.mapchete"), + ) as example: + yield example @pytest.fixture def baselevels_custom_nodata(): """Fixture for baselevels_custom_nodata.mapchete.""" - path = os.path.join(TESTDATA_DIR, "baselevels_custom_nodata.mapchete") - return ExampleConfig(path=path, dict=_dict_from_mapchete(path)) + with ProcessFixture( + os.path.join(TESTDATA_DIR, "baselevels_custom_nodata.mapchete"), + ) as example: + yield example @pytest.fixture def mapchete_input(): """Fixture for mapchete_input.mapchete.""" - path = os.path.join(TESTDATA_DIR, "mapchete_input.mapchete") - return ExampleConfig(path=path, dict=_dict_from_mapchete(path)) + with ProcessFixture( + os.path.join(TESTDATA_DIR, "mapchete_input.mapchete"), + ) as example: + yield example @pytest.fixture def dem_to_hillshade(): """Fixture for dem_to_hillshade.mapchete.""" - path = os.path.join(TESTDATA_DIR, "dem_to_hillshade.mapchete") - return ExampleConfig(path=path, dict=_dict_from_mapchete(path)) + with ProcessFixture( + os.path.join(TESTDATA_DIR, "dem_to_hillshade.mapchete"), + ) as example: + yield example @pytest.fixture def files_bounds(): """Fixture for files_bounds.mapchete.""" - path = os.path.join(TESTDATA_DIR, "files_bounds.mapchete") - return ExampleConfig(path=path, dict=_dict_from_mapchete(path)) + with ProcessFixture( + os.path.join(TESTDATA_DIR, "files_bounds.mapchete"), + ) as example: + yield example @pytest.fixture def example_mapchete(): """Fixture for example.mapchete.""" - path = os.path.join(SCRIPT_DIR, "example.mapchete") - return ExampleConfig(path=path, dict=_dict_from_mapchete(path)) + with ProcessFixture( + os.path.join(SCRIPT_DIR, "example.mapchete"), + ) as example: + yield example @pytest.fixture def example_custom_process_mapchete(): """Fixture for example.mapchete.""" - path = os.path.join(TESTDATA_DIR, "example_custom_process.mapchete") - return ExampleConfig(path=path, dict=_dict_from_mapchete(path)) + with ProcessFixture( + os.path.join(TESTDATA_DIR, "example_custom_process.mapchete"), + ) as example: + yield example @pytest.fixture def zoom_mapchete(): """Fixture for zoom.mapchete.""" - path = os.path.join(TESTDATA_DIR, "zoom.mapchete") - return ExampleConfig(path=path, dict=_dict_from_mapchete(path)) + with ProcessFixture( + os.path.join(TESTDATA_DIR, "zoom.mapchete"), + ) as example: + yield example @pytest.fixture def minmax_zoom(): """Fixture for minmax_zoom.mapchete.""" - path = os.path.join(TESTDATA_DIR, "minmax_zoom.mapchete") - return ExampleConfig(path=path, dict=_dict_from_mapchete(path)) + with ProcessFixture( + os.path.join(TESTDATA_DIR, "minmax_zoom.mapchete"), + ) as example: + yield example @pytest.fixture def cleantopo_tl(): """Fixture for cleantopo_tl.mapchete.""" - path = os.path.join(TESTDATA_DIR, "cleantopo_tl.mapchete") - return ExampleConfig(path=path, dict=_dict_from_mapchete(path)) + with ProcessFixture( + os.path.join(TESTDATA_DIR, "cleantopo_tl.mapchete"), + ) as example: + yield example @pytest.fixture def cleantopo_br(): """Fixture for cleantopo_br.mapchete.""" - path = os.path.join(TESTDATA_DIR, "cleantopo_br.mapchete") - return ExampleConfig(path=path, dict=_dict_from_mapchete(path)) + with ProcessFixture( + os.path.join(TESTDATA_DIR, "cleantopo_br.mapchete"), + ) as example: + yield example @pytest.fixture def cleantopo_br_metatiling_1(): """Fixture for cleantopo_br.mapchete.""" - path = os.path.join(TESTDATA_DIR, "cleantopo_br_metatiling_1.mapchete") - return ExampleConfig(path=path, dict=_dict_from_mapchete(path)) + with ProcessFixture( + os.path.join(TESTDATA_DIR, "cleantopo_br_metatiling_1.mapchete"), + ) as example: + yield example @pytest.fixture def cleantopo_remote(): """Fixture for cleantopo_remote.mapchete.""" - path = os.path.join(TESTDATA_DIR, "cleantopo_remote.mapchete") - return ExampleConfig(path=path, dict=_dict_from_mapchete(path)) + with ProcessFixture( + os.path.join(TESTDATA_DIR, "cleantopo_remote.mapchete"), + ) as example: + yield example @pytest.fixture def cleantopo_br_tiledir(): """Fixture for cleantopo_br_tiledir.mapchete.""" - path = os.path.join(TESTDATA_DIR, "cleantopo_br_tiledir.mapchete") - return ExampleConfig(path=path, dict=_dict_from_mapchete(path)) + with ProcessFixture( + os.path.join(TESTDATA_DIR, "cleantopo_br_tiledir.mapchete"), + ) as example: + yield example @pytest.fixture def cleantopo_br_tiledir_mercator(): """Fixture for cleantopo_br_tiledir_mercator.mapchete.""" - path = os.path.join(TESTDATA_DIR, "cleantopo_br_tiledir_mercator.mapchete") - return ExampleConfig(path=path, dict=_dict_from_mapchete(path)) + with ProcessFixture( + os.path.join(TESTDATA_DIR, "cleantopo_br_tiledir_mercator.mapchete"), + ) as example: + yield example @pytest.fixture def cleantopo_br_mercator(): """Fixture for cleantopo_br_mercator.mapchete.""" - path = os.path.join(TESTDATA_DIR, "cleantopo_br_mercator.mapchete") - return ExampleConfig(path=path, dict=_dict_from_mapchete(path)) + with ProcessFixture( + os.path.join(TESTDATA_DIR, "cleantopo_br_mercator.mapchete"), + ) as example: + yield example @pytest.fixture def geojson(): """Fixture for geojson.mapchete.""" - path = os.path.join(TESTDATA_DIR, "geojson.mapchete") - return ExampleConfig(path=path, dict=_dict_from_mapchete(path)) + with ProcessFixture( + os.path.join(TESTDATA_DIR, "geojson.mapchete"), + ) as example: + yield example @pytest.fixture def geojson_s3(): """Fixture for geojson.mapchete with updated output path.""" - path = os.path.join(TESTDATA_DIR, "geojson.mapchete") - config = _dict_from_mapchete(path) - config["output"].update(path=S3_TEMP_DIR) - return ExampleConfig(path=None, dict=config) + with ProcessFixture( + os.path.join(TESTDATA_DIR, "geojson.mapchete"), + output_tempdir=S3_TEMP_DIR, + ) as example: + yield example @pytest.fixture def geobuf(): """Fixture for geobuf.mapchete.""" - path = os.path.join(TESTDATA_DIR, "geobuf.mapchete") - return ExampleConfig(path=path, dict=_dict_from_mapchete(path)) + with ProcessFixture( + os.path.join(TESTDATA_DIR, "geobuf.mapchete"), + ) as example: + yield example @pytest.fixture def geobuf_s3(): """Fixture for geobuf.mapchete with updated output path.""" - path = os.path.join(TESTDATA_DIR, "geobuf.mapchete") - config = _dict_from_mapchete(path) - config["output"].update(path=S3_TEMP_DIR) - return ExampleConfig(path=None, dict=config) + with ProcessFixture( + os.path.join(TESTDATA_DIR, "geobuf.mapchete"), + output_tempdir=S3_TEMP_DIR, + ) as example: + yield example @pytest.fixture def flatgeobuf(): """Fixture for flatgeobuf.mapchete.""" - path = os.path.join(TESTDATA_DIR, "flatgeobuf.mapchete") - return ExampleConfig(path=path, dict=_dict_from_mapchete(path)) + with ProcessFixture( + os.path.join(TESTDATA_DIR, "flatgeobuf.mapchete"), + ) as example: + yield example @pytest.fixture def flatgeobuf_s3(): """Fixture for flatgeobuf.mapchete with updated output path.""" - path = os.path.join(TESTDATA_DIR, "flatgeobuf.mapchete") - config = _dict_from_mapchete(path) - config["output"].update(path=S3_TEMP_DIR) - return ExampleConfig(path=None, dict=config) + with ProcessFixture( + os.path.join(TESTDATA_DIR, "flatgeobuf.mapchete"), + output_tempdir=S3_TEMP_DIR, + ) as example: + yield example @pytest.fixture def geojson_tiledir(): """Fixture for geojson_tiledir.mapchete.""" - path = os.path.join(TESTDATA_DIR, "geojson_tiledir.mapchete") - return ExampleConfig(path=path, dict=_dict_from_mapchete(path)) + with ProcessFixture( + os.path.join(TESTDATA_DIR, "geojson_tiledir.mapchete"), + ) as example: + yield example @pytest.fixture def process_module(): """Fixture for process_module.mapchete""" - path = os.path.join(TESTDATA_DIR, "process_module.mapchete") - return ExampleConfig(path=path, dict=_dict_from_mapchete(path)) + with ProcessFixture( + os.path.join(TESTDATA_DIR, "process_module.mapchete"), + ) as example: + yield example @pytest.fixture def gtiff_s3(): """Fixture for gtiff_s3.mapchete.""" - path = os.path.join(TESTDATA_DIR, "gtiff_s3.mapchete") - config = _dict_from_mapchete(path) - config["output"].update(path=S3_TEMP_DIR) - return ExampleConfig(path=None, dict=config) + with ProcessFixture( + os.path.join(TESTDATA_DIR, "gtiff_s3.mapchete"), + output_tempdir=S3_TEMP_DIR, + ) as example: + yield example @pytest.fixture def output_single_gtiff(): """Fixture for output_single_gtiff.mapchete.""" - path = os.path.join(TESTDATA_DIR, "output_single_gtiff.mapchete") - return ExampleConfig(path=path, dict=_dict_from_mapchete(path)) + with ProcessFixture( + os.path.join(TESTDATA_DIR, "output_single_gtiff.mapchete"), + ) as example: + yield example @pytest.fixture def output_single_gtiff_cog(): """Fixture for output_single_gtiff_cog.mapchete.""" - path = os.path.join(TESTDATA_DIR, "output_single_gtiff_cog.mapchete") - return ExampleConfig(path=path, dict=_dict_from_mapchete(path)) + with ProcessFixture( + os.path.join(TESTDATA_DIR, "output_single_gtiff_cog.mapchete"), + ) as example: + yield example @pytest.fixture def aoi_br(): """Fixture for aoi_br.mapchete.""" - path = os.path.join(TESTDATA_DIR, "aoi_br.mapchete") - return ExampleConfig(path=path, dict=_dict_from_mapchete(path)) + with ProcessFixture( + os.path.join(TESTDATA_DIR, "aoi_br.mapchete"), + ) as example: + yield example @pytest.fixture def s3_example_tile(gtiff_s3): """Example tile for fixture.""" return (5, 15, 32) - - -# helper functions -def _dict_from_mapchete(path): - config = yaml.safe_load(open(path).read()) - config.update(config_dir=os.path.dirname(path)) - return config diff --git a/test/test_commands.py b/test/test_commands.py index 0432718a..9d8633c5 100644 --- a/test/test_commands.py +++ b/test/test_commands.py @@ -75,27 +75,23 @@ def test_rm(mp_tmpdir, cleantopo_br): assert len(tiles) == 0 -def test_execute(mp_tmpdir, cleantopo_br, cleantopo_br_tif): +def test_execute(mp_tmpdir, cleantopo_br_metatiling_1, cleantopo_br_tif): zoom = 5 - config = cleantopo_br.dict - config["pyramid"].update(metatiling=1) tp = TilePyramid("geodetic") tiles = list(tp.tiles_from_bounds(rasterio.open(cleantopo_br_tif).bounds, zoom)) - job = execute(config, zoom=zoom) + job = execute(cleantopo_br_metatiling_1.dict, zoom=zoom) for t in job: assert t assert len(tiles) == len(job) - with mapchete.open(config) as mp: - for t in tiles: - with rasterio.open(mp.config.output.get_path(t)) as src: - assert not src.read(masked=True).mask.all() + mp = cleantopo_br_metatiling_1.mp() + for t in tiles: + with rasterio.open(mp.config.output.get_path(t)) as src: + assert not src.read(masked=True).mask.all() -def test_execute_cancel(mp_tmpdir, cleantopo_br, cleantopo_br_tif): +def test_execute_cancel(mp_tmpdir, cleantopo_br_metatiling_1, cleantopo_br_tif): zoom = 5 - config = cleantopo_br.dict - config["pyramid"].update(metatiling=1) - job = execute(config, zoom=zoom, as_iterator=True) + job = execute(cleantopo_br_metatiling_1.dict, zoom=zoom, as_iterator=True) for i, t in enumerate(job): job.cancel() break @@ -103,20 +99,18 @@ def test_execute_cancel(mp_tmpdir, cleantopo_br, cleantopo_br_tif): assert job.status == "cancelled" -def test_execute_tile(mp_tmpdir, cleantopo_br): +def test_execute_tile(mp_tmpdir, cleantopo_br_metatiling_1): tile = (5, 30, 63) - config = cleantopo_br.dict - config["pyramid"].update(metatiling=1) - job = execute(config, tile=tile) + job = execute(cleantopo_br_metatiling_1.dict, tile=tile) assert len(job) == 1 - with mapchete.open(config) as mp: - with rasterio.open( - mp.config.output.get_path(mp.config.output_pyramid.tile(*tile)) - ) as src: - assert not src.read(masked=True).mask.all() + mp = cleantopo_br_metatiling_1.mp() + with rasterio.open( + mp.config.output.get_path(mp.config.output_pyramid.tile(*tile)) + ) as src: + assert not src.read(masked=True).mask.all() def test_execute_point(mp_tmpdir, example_mapchete, dummy2_tif): diff --git a/test/test_config.py b/test/test_config.py index 35339852..bf3e3d50 100644 --- a/test/test_config.py +++ b/test/test_config.py @@ -255,12 +255,6 @@ def test_input_zooms(files_zooms): assert os.path.basename(input_files["equals"].path) == "cleantopo_tl.tif" -def test_abstract_input(abstract_input): - """Read abstract input definitions.""" - with pytest.raises(MapcheteDriverError): - MapcheteConfig(abstract_input.path) - - def test_init_zoom(cleantopo_br): with mapchete.open(cleantopo_br.dict, zoom=[3, 5]) as mp: assert mp.config.init_zoom_levels == list(range(3, 6)) diff --git a/test/test_processes.py b/test/test_processes.py index 2433889e..aa3ebdf1 100644 --- a/test/test_processes.py +++ b/test/test_processes.py @@ -6,220 +6,124 @@ import mapchete from mapchete.processes.examples import example_process from mapchete.processes import contours, convert, hillshade +from mapchete.testing import get_process_mp def test_example_process(cleantopo_tl): - with mapchete.open(cleantopo_tl.path) as mp: - zoom = max(mp.config.zoom_levels) - # tile containing data - tile = next(mp.get_process_tiles(zoom)) - user_process = mapchete.MapcheteProcess( - tile=tile, - params=mp.config.params_at_zoom(tile.zoom), - input=mp.config.get_inputs_for_tile(tile), - ) - output = example_process.execute(user_process) - assert isinstance(output, ma.masked_array) - # empty tile - - tile = mp.config.process_pyramid.tile( - zoom, - mp.config.process_pyramid.matrix_height(zoom) - 1, - mp.config.process_pyramid.matrix_width(zoom) - 1, - ) - user_process = mapchete.MapcheteProcess( - tile=tile, - params=mp.config.params_at_zoom(tile.zoom), - input=mp.config.get_inputs_for_tile(tile), - ) - output = example_process.execute(user_process) - assert output == "empty" - - -def test_convert_raster(cleantopo_tl, cleantopo_tl_tif, landpoly): - with mapchete.open(dict(cleantopo_tl.dict, input=dict(inp=cleantopo_tl_tif))) as mp: - zoom = max(mp.config.zoom_levels) - # execute without clip - tile = next(mp.get_process_tiles(zoom)) - user_process = mapchete.MapcheteProcess( - tile=tile, - params=mp.config.params_at_zoom(tile.zoom), - input=mp.config.get_inputs_for_tile(tile), - ) - assert isinstance(convert.execute(user_process), np.ndarray) - # execute on empty tile - tile = mp.config.process_pyramid.tile( - zoom, - mp.config.process_pyramid.matrix_height(zoom) - 1, - mp.config.process_pyramid.matrix_width(zoom) - 1, - ) - user_process = mapchete.MapcheteProcess( - tile=tile, - params=mp.config.params_at_zoom(tile.zoom), - input=mp.config.get_inputs_for_tile(tile), - ) - assert convert.execute(user_process) == "empty" - - with mapchete.open( - dict(cleantopo_tl.dict, input=dict(inp=cleantopo_tl_tif, clip=landpoly)) - ) as mp: - zoom = max(mp.config.zoom_levels) - tile = next(mp.get_process_tiles(zoom)) - user_process = mapchete.MapcheteProcess( - tile=tile, - params=mp.config.params_at_zoom(tile.zoom), - input=mp.config.get_inputs_for_tile(tile), - ) - # tile with data - default = convert.execute(user_process) - assert isinstance(default, np.ndarray) - # scale_offset - offset = convert.execute(user_process, scale_offset=2) - assert isinstance(offset, np.ndarray) - # scale_ratio - ratio = convert.execute(user_process, scale_ratio=0.5) - assert isinstance(ratio, np.ndarray) - # clip_to_output_dtype - clip_dtype = convert.execute( - user_process, scale_ratio=2, clip_to_output_dtype="uint8" - ) - assert isinstance(clip_dtype, np.ndarray) - # execute on empty tile - tile = mp.config.process_pyramid.tile( - zoom, - mp.config.process_pyramid.matrix_height(zoom) - 1, - mp.config.process_pyramid.matrix_width(zoom) - 1, - ) - user_process = mapchete.MapcheteProcess( - tile=tile, - params=mp.config.params_at_zoom(tile.zoom), - input=mp.config.get_inputs_for_tile(tile), - ) - assert convert.execute(user_process) == "empty" - - -def test_convert_vector(cleantopo_tl, landpoly): - with mapchete.open(dict(cleantopo_tl.dict, input=dict(inp=landpoly))) as mp: - zoom = max(mp.config.zoom_levels) - # execute without clip - tile = next(mp.get_process_tiles(zoom)) - user_process = mapchete.MapcheteProcess( - tile=tile, - params=mp.config.params_at_zoom(tile.zoom), - input=mp.config.get_inputs_for_tile(tile), - ) - assert isinstance(convert.execute(user_process), list) - # execute on empty tile - tile = mp.config.process_pyramid.tile( - zoom, - mp.config.process_pyramid.matrix_height(zoom) - 1, - mp.config.process_pyramid.matrix_width(zoom) - 1, - ) - user_process = mapchete.MapcheteProcess( - tile=tile, - params=mp.config.params_at_zoom(tile.zoom), - input=mp.config.get_inputs_for_tile(tile), - ) - assert convert.execute(user_process) == "empty" - - -def test_contours(cleantopo_tl, cleantopo_tl_tif, landpoly): - with mapchete.open(dict(cleantopo_tl.dict, input=dict(dem=cleantopo_tl_tif))) as mp: - zoom = max(mp.config.zoom_levels) - # execute without clip - tile = next(mp.get_process_tiles(zoom)) - user_process = mapchete.MapcheteProcess( - tile=tile, - params=mp.config.params_at_zoom(tile.zoom), - input=mp.config.get_inputs_for_tile(tile), - ) - output = contours.execute(user_process) - assert isinstance(output, list) - assert output - # execute on empty tile - tile = mp.config.process_pyramid.tile( - zoom, - mp.config.process_pyramid.matrix_height(zoom) - 1, - mp.config.process_pyramid.matrix_width(zoom) - 1, - ) - user_process = mapchete.MapcheteProcess( - tile=tile, - params=mp.config.params_at_zoom(tile.zoom), - input=mp.config.get_inputs_for_tile(tile), - ) - assert contours.execute(user_process) == "empty" - - with mapchete.open( - dict(cleantopo_tl.dict, input=dict(dem=cleantopo_tl_tif, clip=landpoly)) - ) as mp: - zoom = max(mp.config.zoom_levels) - tile = next(mp.get_process_tiles(zoom)) - user_process = mapchete.MapcheteProcess( - tile=tile, - params=mp.config.params_at_zoom(tile.zoom), - input=mp.config.get_inputs_for_tile(tile), - ) - output = contours.execute(user_process) - assert isinstance(output, list) - assert output - # execute on empty tile - tile = mp.config.process_pyramid.tile( - zoom, - mp.config.process_pyramid.matrix_height(zoom) - 1, - mp.config.process_pyramid.matrix_width(zoom) - 1, - ) - user_process = mapchete.MapcheteProcess( - tile=tile, - params=mp.config.params_at_zoom(tile.zoom), - input=mp.config.get_inputs_for_tile(tile), - ) - assert contours.execute(user_process) == "empty" - - -def test_hillshade(cleantopo_tl, cleantopo_tl_tif, landpoly): - with mapchete.open(dict(cleantopo_tl.dict, input=dict(dem=cleantopo_tl_tif))) as mp: - zoom = max(mp.config.zoom_levels) - # execute without clip - tile = next(mp.get_process_tiles(zoom)) - user_process = mapchete.MapcheteProcess( - tile=tile, - params=mp.config.params_at_zoom(tile.zoom), - input=mp.config.get_inputs_for_tile(tile), - ) - assert isinstance(hillshade.execute(user_process), np.ndarray) - # execute on empty tile - tile = mp.config.process_pyramid.tile( - zoom, - mp.config.process_pyramid.matrix_height(zoom) - 1, - mp.config.process_pyramid.matrix_width(zoom) - 1, - ) - user_process = mapchete.MapcheteProcess( - tile=tile, - params=mp.config.params_at_zoom(tile.zoom), - input=mp.config.get_inputs_for_tile(tile), - ) - assert hillshade.execute(user_process) == "empty" - - with mapchete.open( - dict(cleantopo_tl.dict, input=dict(dem=cleantopo_tl_tif, clip=landpoly)) - ) as mp: - zoom = max(mp.config.zoom_levels) - tile = next(mp.get_process_tiles(zoom)) - user_process = mapchete.MapcheteProcess( - tile=tile, - params=mp.config.params_at_zoom(tile.zoom), - input=mp.config.get_inputs_for_tile(tile), - ) - assert isinstance(hillshade.execute(user_process), np.ndarray) - # execute on empty tile - tile = mp.config.process_pyramid.tile( - zoom, - mp.config.process_pyramid.matrix_height(zoom) - 1, - mp.config.process_pyramid.matrix_width(zoom) - 1, - ) - user_process = mapchete.MapcheteProcess( - tile=tile, - params=mp.config.params_at_zoom(tile.zoom), - input=mp.config.get_inputs_for_tile(tile), - ) - assert hillshade.execute(user_process) == "empty" + # non-empty tile + output = example_process.execute(cleantopo_tl.process_mp(tile=(5, 0, 0))) + assert isinstance(output, ma.masked_array) + + # empty tile + output = example_process.execute(cleantopo_tl.process_mp(tile=(5, 3, 7))) + assert output == "empty" + + +def test_convert_raster(cleantopo_tl_tif, landpoly): + # tile with data + assert isinstance( + convert.execute(get_process_mp(input=dict(inp=cleantopo_tl_tif), zoom=5)), + np.ndarray, + ) + + # execute on empty tile + assert ( + convert.execute( + get_process_mp(input=dict(inp=cleantopo_tl_tif), tile=(5, 3, 7)) + ) + == "empty" + ) + + process_mp = get_process_mp( + input=dict(inp=cleantopo_tl_tif, clip=landpoly), zoom=5, metatiling=8 + ) + + # tile with data + default = convert.execute(process_mp) + assert isinstance(default, np.ndarray) + + # scale_offset + offset = convert.execute(process_mp, scale_offset=2) + assert isinstance(offset, np.ndarray) + + # scale_ratio + ratio = convert.execute(process_mp, scale_ratio=0.5) + assert isinstance(ratio, np.ndarray) + + # clip_to_output_dtype + clip_dtype = convert.execute( + process_mp, scale_ratio=2, clip_to_output_dtype="uint8" + ) + assert isinstance(clip_dtype, np.ndarray) + + # execute on empty tile + assert ( + convert.execute( + get_process_mp( + input=dict(inp=cleantopo_tl_tif, clip=landpoly), tile=(5, 3, 7) + ) + ) + == "empty" + ) + + +def test_convert_vector(landpoly): + # execute without clip + assert isinstance( + convert.execute(get_process_mp(input=dict(inp=landpoly), zoom=5, metatiling=8)), + list, + ) + + # execute on empty tile + assert ( + convert.execute( + get_process_mp(input=dict(inp=landpoly), tile=(5, 3, 7), metatiling=8) + ) + == "empty" + ) + + +def test_contours(cleantopo_tl_tif, landpoly): + process_mp = get_process_mp(input=dict(dem=cleantopo_tl_tif), zoom=5, metatiling=8) + output = contours.execute(process_mp) + assert isinstance(output, list) + assert output + + # execute on empty tile + process_mp = get_process_mp( + input=dict(dem=cleantopo_tl_tif), tile=(5, 3, 7), metatiling=8 + ) + assert contours.execute(process_mp) == "empty" + + process_mp = get_process_mp( + input=dict(dem=cleantopo_tl_tif, clip=landpoly), zoom=5, metatiling=8 + ) + output = contours.execute(process_mp) + assert isinstance(output, list) + assert output + + process_mp = get_process_mp( + input=dict(dem=cleantopo_tl_tif, clip=landpoly), tile=(5, 3, 7), metatiling=8 + ) + assert contours.execute(process_mp) == "empty" + + +def test_hillshade(cleantopo_tl_tif, landpoly): + process_mp = get_process_mp(input=dict(dem=cleantopo_tl_tif), zoom=5, metatiling=8) + assert isinstance(hillshade.execute(process_mp), np.ndarray) + + # execute on empty tile + process_mp = get_process_mp( + input=dict(dem=cleantopo_tl_tif), tile=(5, 3, 7), metatiling=8 + ) + assert hillshade.execute(process_mp) == "empty" + + process_mp = get_process_mp( + input=dict(dem=cleantopo_tl_tif, clip=landpoly), zoom=5, metatiling=8 + ) + assert isinstance(hillshade.execute(process_mp), np.ndarray) + # execute on empty tile + process_mp = get_process_mp( + input=dict(dem=cleantopo_tl_tif, clip=landpoly), tile=(5, 3, 7), metatiling=8 + ) + assert hillshade.execute(process_mp) == "empty" From 464f26c82885d070e68f5e26502f5e49af008124 Mon Sep 17 00:00:00 2001 From: Joachim Ungar Date: Tue, 2 Nov 2021 20:03:50 +0100 Subject: [PATCH 3/9] fix failing tests --- mapchete/testing.py | 15 +++++++++------ test/test_formats_tiledir_input.py | 1 + 2 files changed, 10 insertions(+), 6 deletions(-) diff --git a/mapchete/testing.py b/mapchete/testing.py index dbf07640..dd7671c8 100644 --- a/mapchete/testing.py +++ b/mapchete/testing.py @@ -43,7 +43,6 @@ def __enter__(self, *args): # set output directory self.dict["output"]["path"] = self._output_tempdir - self._mp = mapchete.open(self.dict) return self def __exit__(self, *args): @@ -54,17 +53,21 @@ def __exit__(self, *args): def clear_output(self): # delete written output if any - out_dir = self.dict["output"]["path"] if self._output_tempdir: - try: - fs_from_path(out_dir).rm(out_dir, recursive=True) - except FileNotFoundError: - pass + out_dir = self._output_tempdir + else: + out_dir = os.path.join(self.dict["config_dir"], self.dict["output"]["path"]) + try: + fs_from_path(out_dir).rm(out_dir, recursive=True) + except FileNotFoundError: + pass def process_mp(self, tile=None, tile_zoom=None): """ Return MapcheteProcess object used to test processes. """ + if not self._mp: + self._mp = mapchete.open(self.dict) if tile: tile = self._mp.config.process_pyramid.tile(*tile) else: diff --git a/test/test_formats_tiledir_input.py b/test/test_formats_tiledir_input.py index 53815185..57eedec6 100644 --- a/test/test_formats_tiledir_input.py +++ b/test/test_formats_tiledir_input.py @@ -59,6 +59,7 @@ def test_read_raster_data(mp_tmpdir, cleantopo_br, cleantopo_br_tiledir): bounds = mp.config.bounds_at_zoom() mp.batch_process(zoom=4) for metatiling in [1, 2, 4, 8]: + cleantopo_br_tiledir.clear_output() _run_tiledir_process_raster(cleantopo_br_tiledir.dict, metatiling, bounds) From f6946fbb3666310c697fdb50839483ac6903682c Mon Sep 17 00:00:00 2001 From: Joachim Ungar Date: Tue, 2 Nov 2021 20:14:10 +0100 Subject: [PATCH 4/9] lazy load Mapchete object --- mapchete/testing.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/mapchete/testing.py b/mapchete/testing.py index dd7671c8..69233388 100644 --- a/mapchete/testing.py +++ b/mapchete/testing.py @@ -66,24 +66,24 @@ def process_mp(self, tile=None, tile_zoom=None): """ Return MapcheteProcess object used to test processes. """ - if not self._mp: - self._mp = mapchete.open(self.dict) if tile: - tile = self._mp.config.process_pyramid.tile(*tile) + tile = self.mp().config.process_pyramid.tile(*tile) else: # just use first process tile from lowest zoom level - zoom = tile_zoom or max(self._mp.config.zoom_levels) - tile = next(self._mp.get_process_tiles(zoom)) + zoom = tile_zoom or max(self.mp().config.zoom_levels) + tile = next(self.mp().get_process_tiles(zoom)) return mapchete.MapcheteProcess( tile=tile, - params=self._mp.config.params_at_zoom(tile.zoom), - input=self._mp.config.get_inputs_for_tile(tile), + params=self.mp().config.params_at_zoom(tile.zoom), + input=self.mp().config.get_inputs_for_tile(tile), ) def mp(self): """ - Return Mapchete object. + Return Mapchete object from mapchete.open(). """ + if not self._mp: + self._mp = mapchete.open(self.dict) return self._mp From d7de8b5a24a9c0821c8af0e054e7ce99daa4dd27 Mon Sep 17 00:00:00 2001 From: Joachim Ungar Date: Tue, 2 Nov 2021 20:41:54 +0100 Subject: [PATCH 5/9] cover no clip test case --- test/test_processes.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/test/test_processes.py b/test/test_processes.py index aa3ebdf1..f40f79c4 100644 --- a/test/test_processes.py +++ b/test/test_processes.py @@ -60,7 +60,9 @@ def test_convert_raster(cleantopo_tl_tif, landpoly): assert ( convert.execute( get_process_mp( - input=dict(inp=cleantopo_tl_tif, clip=landpoly), tile=(5, 3, 7) + input=dict(inp=cleantopo_tl_tif, clip=landpoly), + tile=(5, 0, 0), + metatiling=1, ) ) == "empty" From 02d1a6fd102311843fef39fcbb4f183b0bbddf0d Mon Sep 17 00:00:00 2001 From: Joachim Ungar Date: Tue, 2 Nov 2021 21:02:18 +0100 Subject: [PATCH 6/9] cover more --- mapchete/testing.py | 2 +- test/test_processes.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/mapchete/testing.py b/mapchete/testing.py index 69233388..a4156241 100644 --- a/mapchete/testing.py +++ b/mapchete/testing.py @@ -88,7 +88,7 @@ def mp(self): def get_process_mp(tile=None, zoom=0, input=None, params=None, metatiling=1, **kwargs): - if isinstance(tile, BufferedTile): + if isinstance(tile, BufferedTile): # pragma: no cover pyramid = BufferedTile.tile_pyramid elif isinstance(tile, tuple): pyramid = BufferedTilePyramid("geodetic", metatiling=metatiling) diff --git a/test/test_processes.py b/test/test_processes.py index f40f79c4..7a7150b6 100644 --- a/test/test_processes.py +++ b/test/test_processes.py @@ -11,7 +11,7 @@ def test_example_process(cleantopo_tl): # non-empty tile - output = example_process.execute(cleantopo_tl.process_mp(tile=(5, 0, 0))) + output = example_process.execute(cleantopo_tl.process_mp()) assert isinstance(output, ma.masked_array) # empty tile From 7914379602debbcec432080a5eaf2a12894e15e2 Mon Sep 17 00:00:00 2001 From: Joachim Ungar Date: Wed, 3 Nov 2021 12:39:40 +0100 Subject: [PATCH 7/9] use dictionary configuration where possible --- mapchete/commands/_convert.py | 2 +- test/test_commands.py | 54 +++++++++++++++--------------- test/test_commons.py | 6 ++-- test/test_config.py | 26 +++++++------- test/test_deprecated.py | 2 +- test/test_errors.py | 20 +++++------ test/test_formats.py | 4 +-- test/test_formats_flatgeobuf.py | 4 +-- test/test_formats_geobuf.py | 4 +-- test/test_formats_geojson.py | 4 +-- test/test_formats_geotiff.py | 10 +++--- test/test_formats_tiledir_input.py | 8 ++--- test/test_io.py | 4 +-- test/test_mapchete.py | 40 +++++++++++----------- 14 files changed, 94 insertions(+), 94 deletions(-) diff --git a/mapchete/commands/_convert.py b/mapchete/commands/_convert.py index 99aa7f5b..8d4409c7 100644 --- a/mapchete/commands/_convert.py +++ b/mapchete/commands/_convert.py @@ -30,7 +30,7 @@ def convert( - tiledir: str, + tiledir: Union[str, dict], output: str, zoom: Union[int, List[int]] = None, area: Union[BaseGeometry, str, dict] = None, diff --git a/test/test_commands.py b/test/test_commands.py index e1e0f7aa..faf3fdcf 100644 --- a/test/test_commands.py +++ b/test/test_commands.py @@ -19,7 +19,7 @@ def test_cp(mp_tmpdir, cleantopo_br, wkt_geom): # generate TileDirectory with mapchete.open( - cleantopo_br.path, bounds=[169.19251592399996, -90, 180, -80.18582802550002] + cleantopo_br.dict, bounds=[169.19251592399996, -90, 180, -80.18582802550002] ) as mp: mp.batch_process(zoom=5) out_path = os.path.join(TESTDATA_DIR, cleantopo_br.dict["output"]["path"]) @@ -61,7 +61,7 @@ def test_cp_http(mp_tmpdir, http_tiledir): def test_rm(mp_tmpdir, cleantopo_br): # generate TileDirectory with mapchete.open( - cleantopo_br.path, bounds=[169.19251592399996, -90, 180, -80.18582802550002] + cleantopo_br.dict, bounds=[169.19251592399996, -90, 180, -80.18582802550002] ) as mp: mp.batch_process(zoom=5) out_path = os.path.join(TESTDATA_DIR, cleantopo_br.dict["output"]["path"]) @@ -117,7 +117,7 @@ def test_execute_point(mp_tmpdir, example_mapchete, dummy2_tif): """Using bounds from WKT.""" with rasterio.open(dummy2_tif) as src: g = box(*src.bounds) - job = execute(example_mapchete.path, point=[g.centroid.x, g.centroid.y], zoom=10) + job = execute(example_mapchete.dict, point=[g.centroid.x, g.centroid.y], zoom=10) assert len(job) == 1 @@ -370,7 +370,7 @@ def test_convert_mapchete(cleantopo_br, mp_tmpdir): def test_convert_tiledir(cleantopo_br, mp_tmpdir): # prepare data - with mapchete.open(cleantopo_br.path) as mp: + with mapchete.open(cleantopo_br.dict) as mp: mp.batch_process(zoom=[1, 4]) job = convert( os.path.join( @@ -437,7 +437,7 @@ def test_convert_errors(s2_band_jp2, mp_tmpdir, s2_band, cleantopo_br, landpoly) convert(s2_band, mp_tmpdir) # prepare data for tiledir input - with mapchete.open(cleantopo_br.path) as mp: + with mapchete.open(cleantopo_br.dict) as mp: mp.batch_process(zoom=[1, 4]) tiledir_path = os.path.join( cleantopo_br.dict["config_dir"], cleantopo_br.dict["output"]["path"] @@ -468,11 +468,11 @@ def test_convert_errors(s2_band_jp2, mp_tmpdir, s2_band, cleantopo_br, landpoly) def test_index_geojson(mp_tmpdir, cleantopo_br): # execute process at zoom 3 - job = execute(cleantopo_br.path, zoom=3) + job = execute(cleantopo_br.dict, zoom=3) assert len(job) # generate index for zoom 3 - job = index(cleantopo_br.path, zoom=3, geojson=True) + job = index(cleantopo_br.dict, zoom=3, geojson=True) assert len(job) with mapchete.open(cleantopo_br.dict) as mp: @@ -487,12 +487,12 @@ def test_index_geojson(mp_tmpdir, cleantopo_br): def test_index_geojson_fieldname(mp_tmpdir, cleantopo_br): # execute process at zoom 3 - job = execute(cleantopo_br.path, zoom=3) + job = execute(cleantopo_br.dict, zoom=3) assert len(job) # index and rename "location" to "new_fieldname" job = index( - cleantopo_br.path, + cleantopo_br.dict, zoom=3, geojson=True, fieldname="new_fieldname", @@ -509,12 +509,12 @@ def test_index_geojson_fieldname(mp_tmpdir, cleantopo_br): def test_index_geojson_basepath(mp_tmpdir, cleantopo_br): # execute process at zoom 3 - job = execute(cleantopo_br.path, zoom=3) + job = execute(cleantopo_br.dict, zoom=3) assert len(job) basepath = "http://localhost" # index and rename "location" to "new_fieldname" - job = index(cleantopo_br.path, zoom=3, geojson=True, basepath=basepath) + job = index(cleantopo_br.dict, zoom=3, geojson=True, basepath=basepath) assert len(job) with mapchete.open(cleantopo_br.dict) as mp: @@ -528,13 +528,13 @@ def test_index_geojson_basepath(mp_tmpdir, cleantopo_br): def test_index_geojson_for_gdal(mp_tmpdir, cleantopo_br): # execute process at zoom 3 - job = execute(cleantopo_br.path, zoom=3) + job = execute(cleantopo_br.dict, zoom=3) assert len(job) basepath = "http://localhost" # index and rename "location" to "new_fieldname" job = index( - cleantopo_br.path, zoom=3, geojson=True, basepath=basepath, for_gdal=True + cleantopo_br.dict, zoom=3, geojson=True, basepath=basepath, for_gdal=True ) assert len(job) @@ -549,11 +549,11 @@ def test_index_geojson_for_gdal(mp_tmpdir, cleantopo_br): def test_index_geojson_tile(mp_tmpdir, cleantopo_tl): # execute process at zoom 3 - job = execute(cleantopo_tl.path, zoom=3) + job = execute(cleantopo_tl.dict, zoom=3) assert len(job) # generate index - job = index(cleantopo_tl.path, tile=(3, 0, 0), geojson=True) + job = index(cleantopo_tl.dict, tile=(3, 0, 0), geojson=True) assert len(job) with mapchete.open(cleantopo_tl.dict) as mp: @@ -566,11 +566,11 @@ def test_index_geojson_tile(mp_tmpdir, cleantopo_tl): def test_index_geojson_wkt_area(mp_tmpdir, cleantopo_tl, wkt_geom_tl): # execute process at zoom 3 - job = execute(cleantopo_tl.path, area=wkt_geom_tl) + job = execute(cleantopo_tl.dict, area=wkt_geom_tl) assert len(job) # generate index for zoom 3 - job = index(cleantopo_tl.path, geojson=True, area=wkt_geom_tl) + job = index(cleantopo_tl.dict, geojson=True, area=wkt_geom_tl) assert len(job) with mapchete.open(cleantopo_tl.dict) as mp: @@ -581,11 +581,11 @@ def test_index_geojson_wkt_area(mp_tmpdir, cleantopo_tl, wkt_geom_tl): def test_index_gpkg(mp_tmpdir, cleantopo_br): # execute process - job = execute(cleantopo_br.path, zoom=5) + job = execute(cleantopo_br.dict, zoom=5) assert len(job) # generate index - job = index(cleantopo_br.path, zoom=5, gpkg=True) + job = index(cleantopo_br.dict, zoom=5, gpkg=True) assert len(job) with mapchete.open(cleantopo_br.dict) as mp: files = os.listdir(mp.config.output.path) @@ -596,7 +596,7 @@ def test_index_gpkg(mp_tmpdir, cleantopo_br): assert len(list(src)) == 1 # write again and assert there is no new entry because there is already one - job = index(cleantopo_br.path, zoom=5, gpkg=True) + job = index(cleantopo_br.dict, zoom=5, gpkg=True) assert len(job) with mapchete.open(cleantopo_br.dict) as mp: files = os.listdir(mp.config.output.path) @@ -609,11 +609,11 @@ def test_index_gpkg(mp_tmpdir, cleantopo_br): def test_index_shp(mp_tmpdir, cleantopo_br): # execute process - job = execute(cleantopo_br.path, zoom=5) + job = execute(cleantopo_br.dict, zoom=5) assert len(job) # generate index - job = index(cleantopo_br.path, zoom=5, shp=True) + job = index(cleantopo_br.dict, zoom=5, shp=True) assert len(job) with mapchete.open(cleantopo_br.dict) as mp: files = os.listdir(mp.config.output.path) @@ -624,7 +624,7 @@ def test_index_shp(mp_tmpdir, cleantopo_br): assert len(list(src)) == 1 # write again and assert there is no new entry because there is already one - job = index(cleantopo_br.path, zoom=5, shp=True) + job = index(cleantopo_br.dict, zoom=5, shp=True) assert len(job) with mapchete.open(cleantopo_br.dict) as mp: files = os.listdir(mp.config.output.path) @@ -637,11 +637,11 @@ def test_index_shp(mp_tmpdir, cleantopo_br): def test_index_text(cleantopo_br): # execute process - job = execute(cleantopo_br.path, zoom=5) + job = execute(cleantopo_br.dict, zoom=5) assert len(job) # generate index - job = index(cleantopo_br.path, zoom=5, txt=True) + job = index(cleantopo_br.dict, zoom=5, txt=True) assert len(job) with mapchete.open(cleantopo_br.dict) as mp: files = os.listdir(mp.config.output.path) @@ -653,7 +653,7 @@ def test_index_text(cleantopo_br): assert l.endswith("7.tif\n") # write again and assert there is no new entry because there is already one - job = index(cleantopo_br.path, zoom=5, txt=True) + job = index(cleantopo_br.dict, zoom=5, txt=True) assert len(job) with mapchete.open(cleantopo_br.dict) as mp: files = os.listdir(mp.config.output.path) @@ -667,4 +667,4 @@ def test_index_text(cleantopo_br): def test_index_errors(mp_tmpdir, cleantopo_br): with pytest.raises(ValueError): - index(cleantopo_br.path, zoom=5) + index(cleantopo_br.dict, zoom=5) diff --git a/test/test_commons.py b/test/test_commons.py index 5e811b71..1a2949ab 100644 --- a/test/test_commons.py +++ b/test/test_commons.py @@ -11,7 +11,7 @@ def test_clip(geojson): """Clip an array with a vector.""" - with mapchete.open(geojson.path) as mp: + with mapchete.open(geojson.dict) as mp: tile = next(mp.get_process_tiles(zoom=4)) user_process = MapcheteProcess( tile=tile, @@ -59,7 +59,7 @@ def test_clip(geojson): def test_contours(cleantopo_tl): """Extract contours from array.""" - with mapchete.open(cleantopo_tl.path) as mp: + with mapchete.open(cleantopo_tl.dict) as mp: tile = next(mp.get_process_tiles(zoom=4)) user_process = MapcheteProcess( tile=tile, @@ -84,7 +84,7 @@ def test_contours(cleantopo_tl): def test_hillshade(cleantopo_tl): """Render hillshade from array.""" - with mapchete.open(cleantopo_tl.path) as mp: + with mapchete.open(cleantopo_tl.dict) as mp: tile = next(mp.get_process_tiles(zoom=4)) user_process = MapcheteProcess( tile=tile, diff --git a/test/test_config.py b/test/test_config.py index bf3e3d50..febebad0 100644 --- a/test/test_config.py +++ b/test/test_config.py @@ -73,7 +73,7 @@ def test_config_errors(example_mapchete): def test_config_zoom7(example_mapchete, dummy2_tif): """Example configuration at zoom 5.""" - config = MapcheteConfig(example_mapchete.path) + config = MapcheteConfig(example_mapchete.dict) zoom7 = config.params_at_zoom(7) input_files = zoom7["input"] assert input_files["file1"] is None @@ -86,7 +86,7 @@ def test_config_zoom7(example_mapchete, dummy2_tif): def test_config_zoom11(example_mapchete, dummy2_tif, dummy1_tif): """Example configuration at zoom 11.""" - config = MapcheteConfig(example_mapchete.path) + config = MapcheteConfig(example_mapchete.dict) zoom11 = config.params_at_zoom(11) input_files = zoom11["input"] assert input_files["file1"].path == dummy1_tif @@ -99,41 +99,41 @@ def test_config_zoom11(example_mapchete, dummy2_tif, dummy1_tif): def test_read_zoom_level(zoom_mapchete): """Read zoom level from config file.""" - config = MapcheteConfig(zoom_mapchete.path) + config = MapcheteConfig(zoom_mapchete.dict) assert 5 in config.zoom_levels def test_minmax_zooms(minmax_zoom): """Read min/max zoom levels from config file.""" - config = MapcheteConfig(minmax_zoom.path) + config = MapcheteConfig(minmax_zoom.dict) for zoom in [7, 8, 9, 10]: assert zoom in config.zoom_levels def test_override_zoom_levels(minmax_zoom): """Override zoom levels when constructing configuration.""" - config = MapcheteConfig(minmax_zoom.path, zoom=[7, 8]) + config = MapcheteConfig(minmax_zoom.dict, zoom=[7, 8]) for zoom in [7, 8]: assert zoom in config.zoom_levels def test_read_bounds(zoom_mapchete): """Read bounds from config file.""" - config = MapcheteConfig(zoom_mapchete.path) + config = MapcheteConfig(zoom_mapchete.dict) test_polygon = Polygon([[3, 1.5], [3, 2], [3.5, 2], [3.5, 1.5], [3, 1.5]]) assert config.area_at_zoom(5).equals(test_polygon) def test_override_bounds(zoom_mapchete): """Override bounds when construcing configuration.""" - config = MapcheteConfig(zoom_mapchete.path, bounds=[3, 2, 3.5, 1.5]) + config = MapcheteConfig(zoom_mapchete.dict, bounds=[3, 2, 3.5, 1.5]) test_polygon = Polygon([[3, 1.5], [3, 2], [3.5, 2], [3.5, 1.5], [3, 1.5]]) assert config.area_at_zoom(5).equals(test_polygon) def test_bounds_from_input_files(files_bounds): """Read bounds from input files.""" - config = MapcheteConfig(files_bounds.path) + config = MapcheteConfig(files_bounds.dict) test_polygon = Polygon( [[3, 2], [4, 2], [4, 1], [3, 1], [2, 1], [2, 4], [3, 4], [3, 2]] ) @@ -166,7 +166,7 @@ def test_effective_bounds(files_bounds, baselevels): def test_read_mapchete_input(mapchete_input): """Read Mapchete files as input files.""" - config = MapcheteConfig(mapchete_input.path) + config = MapcheteConfig(mapchete_input.dict) area = config.area_at_zoom(5) # testpolygon = box(0.5, 1.5, 3.5, 3.5) testpolygon = wkt.loads( @@ -177,7 +177,7 @@ def test_read_mapchete_input(mapchete_input): def test_read_baselevels(baselevels): """Read baselevels.""" - config = MapcheteConfig(baselevels.path) + config = MapcheteConfig(baselevels.dict) assert isinstance(config.baselevels, dict) assert set(config.baselevels["zooms"]) == set([5, 6]) assert config.baselevels["lower"] == "bilinear" @@ -205,7 +205,7 @@ def test_empty_input(file_groups): def test_read_input_groups(file_groups): """Read input data groups.""" - config = MapcheteConfig(file_groups.path) + config = MapcheteConfig(file_groups.dict) input_files = config.params_at_zoom(0)["input"] assert "file1" in input_files["group1"] assert "file2" in input_files["group1"] @@ -222,7 +222,7 @@ def test_read_input_groups(file_groups): def test_read_input_order(file_groups): """Assert input objects are represented in the same order as configured.""" - with mapchete.open(file_groups.path) as mp: + with mapchete.open(file_groups.dict) as mp: inputs = yaml.safe_load(open(file_groups.path).read())["input"] tile = mp.config.process_pyramid.tile(0, 0, 0) # read written data from within MapcheteProcess object @@ -236,7 +236,7 @@ def test_read_input_order(file_groups): def test_input_zooms(files_zooms): """Read correct input file per zoom.""" - config = MapcheteConfig(files_zooms.path) + config = MapcheteConfig(files_zooms.dict) # zoom 7 input_files = config.params_at_zoom(7)["input"] assert os.path.basename(input_files["greater_smaller"].path) == "dummy1.tif" diff --git a/test/test_deprecated.py b/test/test_deprecated.py index 16e479d8..c8e4b6d7 100644 --- a/test/test_deprecated.py +++ b/test/test_deprecated.py @@ -44,7 +44,7 @@ def test_deprecated_process_class(deprecated_params): def test_deprecated_open_kwarg(mapchete_input): """Mapchete process as input for other process.""" - with mapchete.open(mapchete_input.path) as mp: + with mapchete.open(mapchete_input.dict) as mp: config = mp.config.params_at_zoom(5) input_data = config["input"]["file2"] with pytest.raises(DeprecationWarning): diff --git a/test/test_errors.py b/test/test_errors.py index 51e714c6..0d0a4db0 100644 --- a/test/test_errors.py +++ b/test/test_errors.py @@ -22,17 +22,17 @@ def test_config_modes(example_mapchete): """Assert process mode is handled correctly.""" # invalid mode with pytest.raises(errors.MapcheteConfigError): - MapcheteConfig(example_mapchete.path, mode="invalid") + MapcheteConfig(example_mapchete.dict, mode="invalid") def test_execute(example_mapchete): """Mapchete execute() errors.""" # in readonly mode - with mapchete.open(example_mapchete.path, mode="readonly") as mp: + with mapchete.open(example_mapchete.dict, mode="readonly") as mp: with pytest.raises(ValueError): mp.execute(next(mp.get_process_tiles())) # wrong tile type - with mapchete.open(example_mapchete.path) as mp: + with mapchete.open(example_mapchete.dict) as mp: with pytest.raises(TypeError): mp.execute("invalid") @@ -40,18 +40,18 @@ def test_execute(example_mapchete): def test_read(example_mapchete): """Mapchete read() errors.""" # in memory mode - with mapchete.open(example_mapchete.path, mode="memory") as mp: + with mapchete.open(example_mapchete.dict, mode="memory") as mp: with pytest.raises(ValueError): mp.read(next(mp.get_process_tiles())) # wrong tile type - with mapchete.open(example_mapchete.path) as mp: + with mapchete.open(example_mapchete.dict) as mp: with pytest.raises(TypeError): mp.read("invalid") def test_write(cleantopo_tl): """Test write function when passing an invalid process_tile.""" - with mapchete.open(cleantopo_tl.path) as mp: + with mapchete.open(cleantopo_tl.dict) as mp: # process and save with pytest.raises(TypeError): mp.write("invalid tile", None) @@ -59,7 +59,7 @@ def test_write(cleantopo_tl): def test_get_raw_output(example_mapchete): """Mapchete get_raw_output() errors.""" - with mapchete.open(example_mapchete.path) as mp: + with mapchete.open(example_mapchete.dict) as mp: # wrong tile type with pytest.raises(TypeError): mp.get_raw_output("invalid") @@ -71,7 +71,7 @@ def test_get_raw_output(example_mapchete): def test_process_tile_write(example_mapchete): """Raise DeprecationWarning on MapcheteProcess.write().""" - config = MapcheteConfig(example_mapchete.path) + config = MapcheteConfig(example_mapchete.dict) tile = BufferedTilePyramid("mercator").tile(7, 1, 1) user_process = mapchete.MapcheteProcess( tile=tile, @@ -84,7 +84,7 @@ def test_process_tile_write(example_mapchete): def test_process_tile_open(example_mapchete): """Raise ValueError on MapcheteProcess.open().""" - config = MapcheteConfig(example_mapchete.path) + config = MapcheteConfig(example_mapchete.dict) tile = BufferedTilePyramid("mercator").tile(7, 1, 1) user_process = mapchete.MapcheteProcess( tile=tile, @@ -97,7 +97,7 @@ def test_process_tile_open(example_mapchete): def test_process_tile_read(example_mapchete): """Raise ValueError on MapcheteProcess.open().""" - config = MapcheteConfig(example_mapchete.path) + config = MapcheteConfig(example_mapchete.dict) tile = BufferedTilePyramid("mercator").tile(7, 1, 1) user_process = mapchete.MapcheteProcess( tile=tile, diff --git a/test/test_formats.py b/test/test_formats.py index ae4764ba..3f831e06 100644 --- a/test/test_formats.py +++ b/test/test_formats.py @@ -69,7 +69,7 @@ def test_driver_from_file_errors(execute_kwargs_py): def test_mapchete_input(mapchete_input): """Mapchete process as input for other process.""" - with mapchete.open(mapchete_input.path) as mp: + with mapchete.open(mapchete_input.dict) as mp: config = mp.config.params_at_zoom(5) input_data = config["input"]["file2"] assert input_data.bbox() @@ -140,7 +140,7 @@ def test_http_rasters(files_bounds, http_raster): def test_read_from_raster_file(cleantopo_br): """Read different bands from source raster.""" - with mapchete.open(cleantopo_br.path) as mp: + with mapchete.open(cleantopo_br.dict) as mp: tile = mp.config.process_pyramid.tile(5, 0, 0) user_process = mapchete.MapcheteProcess( tile=tile, diff --git a/test/test_formats_flatgeobuf.py b/test/test_formats_flatgeobuf.py index 49b2feb9..86d39787 100644 --- a/test/test_formats_flatgeobuf.py +++ b/test/test_formats_flatgeobuf.py @@ -9,7 +9,7 @@ def test_input_data_read(mp_tmpdir, flatgeobuf, landpoly_3857): """Check FlatGeobuf as input data.""" - with mapchete.open(flatgeobuf.path) as mp: + with mapchete.open(flatgeobuf.dict) as mp: for tile in mp.get_process_tiles(): assert isinstance(tile, BufferedTile) input_tile = formats.default.geojson.InputTile(tile, mp) @@ -55,7 +55,7 @@ def test_output_data(mp_tmpdir, flatgeobuf): assert output.file_extension == ".fgb" assert isinstance(output_params, dict) - with mapchete.open(flatgeobuf.path) as mp: + with mapchete.open(flatgeobuf.dict) as mp: tile = mp.config.process_pyramid.tile(4, 3, 7) # write empty mp.write(tile, None) diff --git a/test/test_formats_geobuf.py b/test/test_formats_geobuf.py index 958bb192..0523894e 100644 --- a/test/test_formats_geobuf.py +++ b/test/test_formats_geobuf.py @@ -10,7 +10,7 @@ def test_input_data_read(mp_tmpdir, geobuf, landpoly_3857): """Check Geobuf as input data.""" - with mapchete.open(geobuf.path) as mp: + with mapchete.open(geobuf.dict) as mp: for tile in mp.get_process_tiles(): assert isinstance(tile, BufferedTile) input_tile = formats.default.geojson.InputTile(tile, mp) @@ -80,7 +80,7 @@ def test_output_data(mp_tmpdir, geobuf): assert output.file_extension == ".pbf" assert isinstance(output_params, dict) - with mapchete.open(geobuf.path) as mp: + with mapchete.open(geobuf.dict) as mp: tile = mp.config.process_pyramid.tile(4, 3, 7) # write empty mp.write(tile, None) diff --git a/test/test_formats_geojson.py b/test/test_formats_geojson.py index f054bbc4..64a89fa2 100644 --- a/test/test_formats_geojson.py +++ b/test/test_formats_geojson.py @@ -9,7 +9,7 @@ def test_input_data_read(mp_tmpdir, geojson, landpoly_3857): """Check GeoJSON as input data.""" - with mapchete.open(geojson.path) as mp: + with mapchete.open(geojson.dict) as mp: for tile in mp.get_process_tiles(): assert isinstance(tile, BufferedTile) input_tile = formats.default.geojson.InputTile(tile, mp) @@ -76,7 +76,7 @@ def test_output_data(mp_tmpdir, geojson): assert output.file_extension == ".geojson" assert isinstance(output_params, dict) - with mapchete.open(geojson.path) as mp: + with mapchete.open(geojson.dict) as mp: tile = mp.config.process_pyramid.tile(4, 3, 7) # write empty mp.write(tile, None) diff --git a/test/test_formats_geotiff.py b/test/test_formats_geotiff.py index b92548f1..f294ab03 100644 --- a/test/test_formats_geotiff.py +++ b/test/test_formats_geotiff.py @@ -129,7 +129,7 @@ def test_for_web(client, mp_tmpdir): def test_input_data(mp_tmpdir, cleantopo_br): """Check GeoTIFF proces output as input data.""" - with mapchete.open(cleantopo_br.path) as mp: + with mapchete.open(cleantopo_br.dict) as mp: tp = BufferedTilePyramid("geodetic") # TODO tile with existing but empty data tile = tp.tile(5, 5, 5) @@ -205,7 +205,7 @@ def test_s3_write_output_data(gtiff_s3, s3_example_tile, mp_s3_tmpdir): def test_output_single_gtiff(output_single_gtiff): tile_id = (5, 3, 7) - with mapchete.open(output_single_gtiff.path) as mp: + with mapchete.open(output_single_gtiff.dict) as mp: process_tile = mp.config.process_pyramid.tile(*tile_id) # basic functions assert mp.config.output.profile() @@ -231,10 +231,10 @@ def test_output_single_gtiff(output_single_gtiff): # error on existing file with pytest.raises(MapcheteConfigError): - mapchete.open(output_single_gtiff.path) + mapchete.open(output_single_gtiff.dict) # overwrite existing file - with mapchete.open(output_single_gtiff.path, mode="overwrite") as mp: + with mapchete.open(output_single_gtiff.dict, mode="overwrite") as mp: process_tile = mp.config.process_pyramid.tile(*tile_id) assert not mp.config.output.tiles_exist(process_tile) # write @@ -256,7 +256,7 @@ def test_output_single_gtiff_errors(output_single_gtiff): mapchete.open(dict(output_single_gtiff.dict, zoom_levels=[5, 6])) # provide either process_tile or output_tile - with mapchete.open(output_single_gtiff.path) as mp: + with mapchete.open(output_single_gtiff.dict) as mp: tile = mp.config.process_pyramid.tile(5, 3, 7) with pytest.raises(ValueError): mp.config.output.tiles_exist(process_tile=tile, output_tile=tile) diff --git a/test/test_formats_tiledir_input.py b/test/test_formats_tiledir_input.py index 57eedec6..c43bb2af 100644 --- a/test/test_formats_tiledir_input.py +++ b/test/test_formats_tiledir_input.py @@ -33,7 +33,7 @@ def test_parse_bounds(geojson_tiledir): def test_read_vector_data(mp_tmpdir, geojson, geojson_tiledir): """Read vector data.""" # prepare data - with mapchete.open(geojson.path) as mp: + with mapchete.open(geojson.dict) as mp: bounds = mp.config.bounds_at_zoom() mp.batch_process(zoom=4) # read data @@ -55,7 +55,7 @@ def _run_tiledir_process_vector(conf_dict, metatiling, bounds): def test_read_raster_data(mp_tmpdir, cleantopo_br, cleantopo_br_tiledir): """Read raster data.""" # prepare data - with mapchete.open(cleantopo_br.path) as mp: + with mapchete.open(cleantopo_br.dict) as mp: bounds = mp.config.bounds_at_zoom() mp.batch_process(zoom=4) for metatiling in [1, 2, 4, 8]: @@ -89,7 +89,7 @@ def test_read_reprojected_raster_data( """Read reprojected raster data.""" zoom = 4 # prepare data - with mapchete.open(cleantopo_br.path) as mp: + with mapchete.open(cleantopo_br.dict) as mp: mp.batch_process(zoom=zoom) with mapchete.open(cleantopo_br_tiledir_mercator.dict, mode="overwrite") as mp: @@ -142,7 +142,7 @@ def test_read_reprojected_raster_data( def test_read_from_dir(mp_tmpdir, cleantopo_br, cleantopo_br_tiledir): """Read raster data.""" # prepare data - with mapchete.open(cleantopo_br.path) as mp: + with mapchete.open(cleantopo_br.dict) as mp: bounds = mp.config.bounds_at_zoom() mp.batch_process(zoom=4) config = dict(cleantopo_br_tiledir.dict, input=dict(file1="tmp/cleantopo_br")) diff --git a/test/test_io.py b/test/test_io.py index fb9e2078..b81b61f3 100644 --- a/test/test_io.py +++ b/test/test_io.py @@ -64,7 +64,7 @@ def test_read_raster_window(dummy1_tif, minmax_zoom): """Read array with read_raster_window.""" zoom = 8 # without reproject - config = MapcheteConfig(minmax_zoom.path) + config = MapcheteConfig(minmax_zoom.dict) rasterfile = config.params_at_zoom(zoom)["input"]["file1"] dummy1_bbox = rasterfile.bbox() @@ -775,7 +775,7 @@ def test_prepare_array_errors(): def test_read_vector_window(geojson, landpoly_3857): """Read vector data from read_vector_window.""" zoom = 4 - config = MapcheteConfig(geojson.path) + config = MapcheteConfig(geojson.dict) vectorfile = config.params_at_zoom(zoom)["input"]["file1"] pixelbuffer = 5 tile_pyramid = BufferedTilePyramid("geodetic", pixelbuffer=pixelbuffer) diff --git a/test/test_mapchete.py b/test/test_mapchete.py index 6c467516..a683b48b 100644 --- a/test/test_mapchete.py +++ b/test/test_mapchete.py @@ -26,14 +26,14 @@ def test_empty_execute(mp_tmpdir, cleantopo_br): """Execute process outside of defined zoom levels.""" - with mapchete.open(cleantopo_br.path) as mp: + with mapchete.open(cleantopo_br.dict) as mp: assert mp.execute((6, 0, 0)).mask.all() def test_read_existing_output(mp_tmpdir, cleantopo_tl): """Read existing process output.""" # raster data - with mapchete.open(cleantopo_tl.path) as mp: + with mapchete.open(cleantopo_tl.dict) as mp: tile = mp.config.process_pyramid.tile(5, 0, 0) # process and save mp.get_raw_output(tile) @@ -65,7 +65,7 @@ def test_read_existing_output_buffer(mp_tmpdir, cleantopo_tl): def test_read_existing_output_vector(mp_tmpdir, geojson): """Read existing process output with process buffer.""" - with mapchete.open(geojson.path) as mp: + with mapchete.open(geojson.dict) as mp: tile = next(mp.get_process_tiles(4)) # process and save mp.write(tile, mp.get_raw_output(tile)) @@ -84,7 +84,7 @@ def test_read_existing_output_vector(mp_tmpdir, geojson): def test_open_data_error(cleantopo_tl): """Try to open data not specified as input.""" - with mapchete.open(cleantopo_tl.path) as mp: + with mapchete.open(cleantopo_tl.dict) as mp: tile = mp.config.process_pyramid.tile(5, 0, 0) # read written data from within MapcheteProcess object user_process = mapchete.MapcheteProcess( @@ -97,13 +97,13 @@ def test_open_data_error(cleantopo_tl): def test_get_raw_output_outside(mp_tmpdir, cleantopo_br): """Get raw process output outside of zoom levels.""" - with mapchete.open(cleantopo_br.path) as mp: + with mapchete.open(cleantopo_br.dict) as mp: assert mp.get_raw_output((6, 0, 0)).mask.all() def test_get_raw_output_memory(mp_tmpdir, cleantopo_tl): """Get raw process output using memory flag.""" - with mapchete.open(cleantopo_tl.path, mode="memory") as mp: + with mapchete.open(cleantopo_tl.dict, mode="memory") as mp: assert mp.config.mode == "memory" assert not mp.get_raw_output((5, 0, 0)).mask.all() @@ -111,8 +111,8 @@ def test_get_raw_output_memory(mp_tmpdir, cleantopo_tl): def test_get_raw_output_readonly(mp_tmpdir, cleantopo_tl): """Get raw process output using readonly flag.""" tile = (5, 0, 0) - readonly_mp = mapchete.open(cleantopo_tl.path, mode="readonly") - write_mp = mapchete.open(cleantopo_tl.path, mode="continue") + readonly_mp = mapchete.open(cleantopo_tl.dict, mode="readonly") + write_mp = mapchete.open(cleantopo_tl.dict, mode="continue") # read non-existing data (returns empty) assert readonly_mp.get_raw_output(tile).mask.all() @@ -130,7 +130,7 @@ def test_get_raw_output_readonly(mp_tmpdir, cleantopo_tl): def test_get_raw_output_continue_raster(mp_tmpdir, cleantopo_tl): """Get raw process output using continue flag.""" - with mapchete.open(cleantopo_tl.path) as mp: + with mapchete.open(cleantopo_tl.dict) as mp: assert mp.config.mode == "continue" tile = (5, 0, 0) # process and save @@ -141,7 +141,7 @@ def test_get_raw_output_continue_raster(mp_tmpdir, cleantopo_tl): def test_get_raw_output_continue_vector(mp_tmpdir, geojson): """Get raw process output using continue flag.""" - with mapchete.open(geojson.path) as mp: + with mapchete.open(geojson.dict) as mp: assert mp.config.mode == "continue" tile = next(mp.get_process_tiles(4)) # process and save @@ -161,7 +161,7 @@ def test_get_raw_output_continue_vector(mp_tmpdir, geojson): def test_baselevels(mp_tmpdir, baselevels): """Baselevel interpolation.""" - with mapchete.open(baselevels.path, mode="continue") as mp: + with mapchete.open(baselevels.dict, mode="continue") as mp: # process data before getting baselevels mp.batch_process() @@ -189,7 +189,7 @@ def test_baselevels(mp_tmpdir, baselevels): def test_baselevels_custom_nodata(mp_tmpdir, baselevels_custom_nodata): """Baselevel interpolation.""" fill_value = -32768.0 - with mapchete.open(baselevels_custom_nodata.path, mode="continue") as mp: + with mapchete.open(baselevels_custom_nodata.dict, mode="continue") as mp: # process data before getting baselevels mp.batch_process() @@ -353,7 +353,7 @@ def test_baselevels_buffer_antimeridian(mp_tmpdir, baselevels): def test_processing(mp_tmpdir, cleantopo_br, cleantopo_tl): """Test correct processing (read and write) outputs.""" - for cleantopo_process in [cleantopo_br.path, cleantopo_tl.path]: + for cleantopo_process in [cleantopo_br.dict, cleantopo_tl.dict]: with mapchete.open(cleantopo_process) as mp: for zoom in range(6): tiles = [] @@ -388,7 +388,7 @@ def test_processing(mp_tmpdir, cleantopo_br, cleantopo_tl): def test_pickleability(mp_tmpdir, cleantopo_tl): """Test parallel tile processing.""" - with mapchete.open(cleantopo_tl.path) as mp: + with mapchete.open(cleantopo_tl.dict) as mp: assert pickle_dumps(mp) assert pickle_dumps(mp.config) assert pickle_dumps(mp.config.output) @@ -403,7 +403,7 @@ def _worker(mp, tile): def test_write_empty(mp_tmpdir, cleantopo_tl): """Test write function when passing an empty process_tile.""" - with mapchete.open(cleantopo_tl.path) as mp: + with mapchete.open(cleantopo_tl.dict) as mp: # process and save mp.write(mp.config.process_pyramid.tile(5, 0, 0), None) @@ -474,7 +474,7 @@ def test_count_tiles_large_init_zoom(geometrycollection): def test_batch_process(mp_tmpdir, cleantopo_tl): """Test batch_process function.""" - with mapchete.open(cleantopo_tl.path) as mp: + with mapchete.open(cleantopo_tl.dict) as mp: # invalid parameters errors with pytest.raises(ValueError): mp.batch_process(zoom=1, tile=(1, 0, 0)) @@ -489,12 +489,12 @@ def test_batch_process(mp_tmpdir, cleantopo_tl): def test_skip_tiles(mp_tmpdir, cleantopo_tl): """Test batch_process function.""" zoom = 2 - with mapchete.open(cleantopo_tl.path, mode="continue") as mp: + with mapchete.open(cleantopo_tl.dict, mode="continue") as mp: mp.batch_process(zoom=zoom) for tile, skip in mp.skip_tiles(tiles=mp.get_process_tiles(zoom=zoom)): assert skip - with mapchete.open(cleantopo_tl.path, mode="overwrite") as mp: + with mapchete.open(cleantopo_tl.dict, mode="overwrite") as mp: for tile, skip in mp.skip_tiles(tiles=mp.get_process_tiles(zoom=zoom)): assert not skip @@ -502,10 +502,10 @@ def test_skip_tiles(mp_tmpdir, cleantopo_tl): def test_custom_grid(mp_tmpdir, custom_grid): """Cutom grid processing.""" # process and save - with mapchete.open(custom_grid.path) as mp: + with mapchete.open(custom_grid.dict) as mp: mp.batch_process() # read written output - with mapchete.open(custom_grid.path) as mp: + with mapchete.open(custom_grid.dict) as mp: for tile in mp.get_process_tiles(5): data = mp.config.output.read(tile) assert data.any() From 938ddaf39c50147f90a9c8e625a1aa47ced3423e Mon Sep 17 00:00:00 2001 From: Joachim Ungar Date: Wed, 3 Nov 2021 13:49:42 +0100 Subject: [PATCH 8/9] use proper s3 tempdir fixtures --- mapchete/testing.py | 6 ++++-- test/README.rst | 2 +- test/conftest.py | 22 ++++++++++++++++++++++ test/test_formats_flatgeobuf.py | 8 ++++---- test/test_formats_geobuf.py | 8 ++++---- test/test_formats_geojson.py | 8 ++++---- test/test_formats_geotiff.py | 33 ++++++++------------------------- test/test_index.py | 2 +- test/test_io.py | 2 +- 9 files changed, 49 insertions(+), 42 deletions(-) diff --git a/mapchete/testing.py b/mapchete/testing.py index a4156241..8ecbc9ee 100644 --- a/mapchete/testing.py +++ b/mapchete/testing.py @@ -27,11 +27,13 @@ def dict_from_mapchete(path): class ProcessFixture: - def __init__(self, path=None, output_tempdir=None): + def __init__(self, path=None, output_tempdir=None, output_suffix=""): self.path = path self.dict = None if output_tempdir: - self._output_tempdir = os.path.join(output_tempdir, uuid.uuid4().hex) + self._output_tempdir = ( + os.path.join(output_tempdir, uuid.uuid4().hex) + output_suffix + ) else: self._output_tempdir = None self._out_fs = None diff --git a/test/README.rst b/test/README.rst index 84d83053..6b3a788d 100644 --- a/test/README.rst +++ b/test/README.rst @@ -4,7 +4,7 @@ Run tests .. code-block:: shell - py.test test_*.py + pytest -v --cov mapchete Under certain environments, curl-based tests can fail. In this case, try: diff --git a/test/conftest.py b/test/conftest.py index 7211bfd3..fb7738e3 100644 --- a/test/conftest.py +++ b/test/conftest.py @@ -564,6 +564,17 @@ def output_single_gtiff(): yield example +@pytest.fixture +def output_single_gtiff_s3(): + """Fixture for output_single_gtiff.mapchete.""" + with ProcessFixture( + os.path.join(TESTDATA_DIR, "output_single_gtiff.mapchete"), + output_tempdir=S3_TEMP_DIR, + output_suffix=".tif", + ) as example: + yield example + + @pytest.fixture def output_single_gtiff_cog(): """Fixture for output_single_gtiff_cog.mapchete.""" @@ -573,6 +584,17 @@ def output_single_gtiff_cog(): yield example +@pytest.fixture +def output_single_gtiff_cog_s3(): + """Fixture for output_single_gtiff_cog.mapchete.""" + with ProcessFixture( + os.path.join(TESTDATA_DIR, "output_single_gtiff_cog.mapchete"), + output_tempdir=S3_TEMP_DIR, + output_suffix=".tif", + ) as example: + yield example + + @pytest.fixture def aoi_br(): """Fixture for aoi_br.mapchete.""" diff --git a/test/test_formats_flatgeobuf.py b/test/test_formats_flatgeobuf.py index 86d39787..c5c7de83 100644 --- a/test/test_formats_flatgeobuf.py +++ b/test/test_formats_flatgeobuf.py @@ -69,24 +69,24 @@ def test_output_data(mp_tmpdir, flatgeobuf): @pytest.mark.remote -def test_s3_output_data(mp_s3_tmpdir, flatgeobuf_s3): +def test_s3_output_data(flatgeobuf_s3): """Check FlatGeobuf as output data.""" output_params = dict( grid="geodetic", format="FlatGeobuf", - path=mp_s3_tmpdir, + path=flatgeobuf_s3.dict["output"]["path"], schema=dict(properties=dict(id="int"), geometry="Polygon"), pixelbuffer=0, metatiling=1, ) output = formats.default.flatgeobuf.OutputDataWriter(output_params) - assert output.path == mp_s3_tmpdir + assert output.path == flatgeobuf_s3.dict["output"]["path"] assert output.file_extension == ".fgb" assert isinstance(output_params, dict) @pytest.mark.remote -def test_s3_output_data_rw(mp_s3_tmpdir, flatgeobuf_s3): +def test_s3_output_data_rw(flatgeobuf_s3): with mapchete.open(flatgeobuf_s3.dict) as mp: tile = mp.config.process_pyramid.tile(4, 3, 7) # write empty diff --git a/test/test_formats_geobuf.py b/test/test_formats_geobuf.py index 0523894e..22989768 100644 --- a/test/test_formats_geobuf.py +++ b/test/test_formats_geobuf.py @@ -94,24 +94,24 @@ def test_output_data(mp_tmpdir, geobuf): @pytest.mark.remote -def test_s3_output_data(mp_s3_tmpdir, geobuf_s3): +def test_s3_output_data(geobuf_s3): """Check Geobuf as output data.""" output_params = dict( grid="geodetic", format="Geobuf", - path=mp_s3_tmpdir, + path=geobuf_s3.dict["output"]["path"], schema=dict(properties=dict(id="int"), geometry="Polygon"), pixelbuffer=0, metatiling=1, ) output = formats.default.geobuf.OutputDataWriter(output_params) - assert output.path == mp_s3_tmpdir + assert output.path == geobuf_s3.dict["output"]["path"] assert output.file_extension == ".pbf" assert isinstance(output_params, dict) @pytest.mark.remote -def test_s3_output_data_rw(mp_s3_tmpdir, geobuf_s3): +def test_s3_output_data_rw(geobuf_s3): with mapchete.open(geobuf_s3.dict) as mp: tile = mp.config.process_pyramid.tile(4, 3, 7) # write empty diff --git a/test/test_formats_geojson.py b/test/test_formats_geojson.py index 64a89fa2..f4a227ec 100644 --- a/test/test_formats_geojson.py +++ b/test/test_formats_geojson.py @@ -90,24 +90,24 @@ def test_output_data(mp_tmpdir, geojson): @pytest.mark.remote -def test_s3_output_data(mp_s3_tmpdir, geojson_s3): +def test_s3_output_data(geojson_s3): """Check GeoJSON as output data.""" output_params = dict( grid="geodetic", format="GeoJSON", - path=mp_s3_tmpdir, + path=geojson_s3.dict["output"]["path"], schema=dict(properties=dict(id="int"), geometry="Polygon"), pixelbuffer=0, metatiling=1, ) output = formats.default.geojson.OutputDataWriter(output_params) - assert output.path == mp_s3_tmpdir + assert output.path == geojson_s3.dict["output"]["path"] assert output.file_extension == ".geojson" assert isinstance(output_params, dict) @pytest.mark.remote -def test_s3_output_data_rw(mp_s3_tmpdir, geojson_s3): +def test_s3_output_data_rw(geojson_s3): with mapchete.open(geojson_s3.dict) as mp: tile = mp.config.process_pyramid.tile(4, 3, 7) # write empty diff --git a/test/test_formats_geotiff.py b/test/test_formats_geotiff.py index f294ab03..67fccf65 100644 --- a/test/test_formats_geotiff.py +++ b/test/test_formats_geotiff.py @@ -183,7 +183,7 @@ def test_write_geotiff_tags(mp_tmpdir, cleantopo_br, write_rasterfile_tags_py): @pytest.mark.remote -def test_s3_write_output_data(gtiff_s3, s3_example_tile, mp_s3_tmpdir): +def test_s3_write_output_data(gtiff_s3, s3_example_tile): """Write and read output.""" with mapchete.open(gtiff_s3.dict) as mp: process_tile = mp.config.process_pyramid.tile(*s3_example_tile) @@ -331,17 +331,9 @@ def test_output_single_gtiff_overviews(output_single_gtiff): @pytest.mark.remote -def test_output_single_gtiff_s3(output_single_gtiff, mp_s3_tmpdir): +def test_output_single_gtiff_s3(output_single_gtiff_s3): tile_id = (5, 3, 7) - with mapchete.open( - dict( - output_single_gtiff.dict, - output=dict( - output_single_gtiff.dict["output"], - path=os.path.join(mp_s3_tmpdir, "temp.tif"), - ), - ) - ) as mp: + with mapchete.open(output_single_gtiff_s3.dict) as mp: process_tile = mp.config.process_pyramid.tile(*tile_id) # basic functions assert mp.config.output.profile() @@ -367,14 +359,13 @@ def test_output_single_gtiff_s3(output_single_gtiff, mp_s3_tmpdir): @pytest.mark.remote -def test_output_single_gtiff_s3_tempfile(output_single_gtiff, mp_s3_tmpdir): +def test_output_single_gtiff_s3_tempfile(output_single_gtiff_s3): tile_id = (5, 3, 7) with mapchete.open( dict( - output_single_gtiff.dict, + output_single_gtiff_s3.dict, output=dict( - output_single_gtiff.dict["output"], - path=os.path.join(mp_s3_tmpdir, "temp.tif"), + output_single_gtiff_s3.dict["output"], in_memory=False, ), ) @@ -474,17 +465,9 @@ def test_output_single_gtiff_cog_tempfile(output_single_gtiff_cog): @pytest.mark.skipif( not GDAL_COG_AVAILABLE, reason="GDAL>=3.1 with COG driver is required" ) -def test_output_single_gtiff_cog_s3(output_single_gtiff_cog, mp_s3_tmpdir): +def test_output_single_gtiff_cog_s3(output_single_gtiff_cog_s3): tile_id = (5, 3, 7) - with mapchete.open( - dict( - output_single_gtiff_cog.dict, - output=dict( - output_single_gtiff_cog.dict["output"], - path=os.path.join(mp_s3_tmpdir, "cog.tif"), - ), - ) - ) as mp: + with mapchete.open(output_single_gtiff_cog_s3.dict) as mp: process_tile = mp.config.process_pyramid.tile(*tile_id) # basic functions assert mp.config.output.profile() diff --git a/test/test_index.py b/test/test_index.py index acfc20f2..94af907c 100644 --- a/test/test_index.py +++ b/test/test_index.py @@ -10,7 +10,7 @@ @pytest.mark.remote -def test_remote_indexes(mp_s3_tmpdir, gtiff_s3): +def test_remote_indexes(gtiff_s3): zoom = 7 gtiff_s3.dict.update(zoom_levels=zoom) diff --git a/test/test_io.py b/test/test_io.py index b81b61f3..95790378 100644 --- a/test/test_io.py +++ b/test/test_io.py @@ -1139,7 +1139,7 @@ def test_tiles_exist_local(example_mapchete): assert set(output_tiles) == existing.union(not_existing) -def test_tiles_exist_s3(gtiff_s3, mp_s3_tmpdir): +def test_tiles_exist_s3(gtiff_s3): bounds = (0, 0, 10, 10) # bounds = (3, 1, 4, 2) zoom = 5 From b2377a5db7b723d1d2840ecc3f83095a5de17f95 Mon Sep 17 00:00:00 2001 From: Joachim Ungar Date: Wed, 3 Nov 2021 16:10:39 +0100 Subject: [PATCH 9/9] add first_process_tile() function --- mapchete/testing.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/mapchete/testing.py b/mapchete/testing.py index 8ecbc9ee..820ef8bb 100644 --- a/mapchete/testing.py +++ b/mapchete/testing.py @@ -72,8 +72,7 @@ def process_mp(self, tile=None, tile_zoom=None): tile = self.mp().config.process_pyramid.tile(*tile) else: # just use first process tile from lowest zoom level - zoom = tile_zoom or max(self.mp().config.zoom_levels) - tile = next(self.mp().get_process_tiles(zoom)) + tile = self.first_process_tile(zoom=tile_zoom) return mapchete.MapcheteProcess( tile=tile, params=self.mp().config.params_at_zoom(tile.zoom), @@ -88,11 +87,13 @@ def mp(self): self._mp = mapchete.open(self.dict) return self._mp + def first_process_tile(self, zoom=None): + zoom = zoom or max(self.mp().config.zoom_levels) + return next(self.mp().get_process_tiles(zoom)) + def get_process_mp(tile=None, zoom=0, input=None, params=None, metatiling=1, **kwargs): - if isinstance(tile, BufferedTile): # pragma: no cover - pyramid = BufferedTile.tile_pyramid - elif isinstance(tile, tuple): + if tile: pyramid = BufferedTilePyramid("geodetic", metatiling=metatiling) tile = pyramid.tile(*tile) else: