Merge pull request #620 from ungarj/mpath_cli

add mpath cli

mapchete/cli/mpath.py

@@ -0,0 +1,185 @@
+import json
+from typing import List, Optional, Union
+
+import click
+
+from mapchete.cli import options
+from mapchete.io import copy
+from mapchete.path import MPath
+from mapchete.pretty import pretty_bytes
+
+opt_recursive = click.option("--recursive", "-r", is_flag=True)
+
+
+@click.group()
+def mpath():
+    pass
+
+
+@mpath.command(help="Check whether path exists.")
+@options.arg_path
+def exists(path: MPath):
+    click.echo(path.exists())
+
+
+@mpath.command(help="copy path.")
+@options.arg_path
+@options.arg_out_path
+@options.opt_overwrite
+def cp(path: MPath, out_path: MPath, overwrite: bool = False):
+    try:
+        copy(path, out_path, overwrite=overwrite)
+    except Exception as exc:  # pragma: no cover
+        raise click.ClickException(exc)
+
+
+@mpath.command(help="Remove path.")
+@options.arg_path
+@opt_recursive
+@options.opt_force
+def rm(path: MPath, recursive: bool = False, force: bool = False):
+    try:
+        if force or click.confirm(
+            f"do you really want to permanently delete {str(path)}?"
+        ):
+            path.rm(recursive=recursive)
+    except Exception as exc:  # pragma: no cover
+        raise click.ClickException(exc)
+
+
+@mpath.command(help="List path contents.")
+@options.arg_path
+@click.option(
+    "--date-format", type=click.STRING, default="%y-%m-%d %H:%M:%S", show_default=True
+)
+@click.option("--absolute-paths", is_flag=True)
+@click.option("--spacing", type=click.INT, default=4, show_default=True)
+@click.option("--max-depth", type=click.INT, default=None, show_default=True)
+@opt_recursive
+def ls(
+    path: MPath,
+    date_format: str = "%y-%m-%d %H:%M:%S",
+    absolute_paths: bool = False,
+    spacing: int = 4,
+    recursive: bool = False,
+    max_depth: Optional[int] = None,
+):
+    size_column_width = 10
+
+    def _print_rows(
+        directories: List[MPath],
+        files: List[MPath],
+        last_modified_column_width: int = 0,
+        size_column_width: int = 10,
+        spacing: int = 4,
+    ):
+        for subpath in directories:
+            click.echo(
+                _row(
+                    columns=[
+                        "",
+                        "",
+                        f"{str(subpath)}/"
+                        if absolute_paths
+                        else f"{subpath.relative_to(path)}/",
+                    ],
+                    widths=[last_modified_column_width, size_column_width, None],
+                    spacing=spacing,
+                )
+            )
+        for subpath in files:
+            click.echo(
+                _row(
+                    columns=[
+                        subpath.last_modified().strftime(date_format),
+                        pretty_bytes(subpath.size()),
+                        str(subpath) if absolute_paths else subpath.relative_to(path),
+                    ],
+                    widths=[last_modified_column_width, size_column_width, None],
+                    spacing=spacing,
+                )
+            )
+
+    try:
+        last_modified_column_width = len(date_format)
+        click.echo(
+            _row(
+                columns=["last modified", "size", "path"],
+                widths=[last_modified_column_width, size_column_width, None],
+                spacing=spacing,
+                underlines=True,
+            )
+        )
+        if recursive:
+            for root, _, files in path.walk(absolute_paths=True, maxdepth=max_depth):
+                _print_rows(
+                    [root],
+                    files,
+                    last_modified_column_width=last_modified_column_width,
+                    spacing=spacing,
+                )
+        else:
+            directories = []
+            files = []
+            for subpath in path.ls(absolute_paths=True):
+                if subpath.is_directory():
+                    directories.append(subpath)
+                else:
+                    files.append(subpath)
+            _print_rows(
+                directories,
+                files,
+                last_modified_column_width=last_modified_column_width,
+                spacing=spacing,
+            )
+
+    except Exception as exc:  # pragma: no cover
+        raise click.ClickException(exc)
+
+
+def _row(
+    columns: List[str],
+    widths: List[Union[int, None]],
+    spacing: int = 4,
+    underlines: bool = False,
+) -> str:
+    def _column(text: str = "", width: Optional[int] = None):
+        if width is None or len(text) > width:
+            width = len(text)
+        return text + " " * (width - len(text))
+
+    def _column_underline(text: str = "", width: int = 4, symbol: str = "-"):
+        if width is None or len(text) > width:
+            width = len(text)
+        return symbol * len(text) + " " * (width - len(text))
+
+    space = " " * spacing
+
+    out = space.join([_column(column, width) for column, width in zip(columns, widths)])
+
+    if underlines:
+        out += "\n"
+        out += space.join(
+            [_column_underline(column, width) for column, width in zip(columns, widths)]
+        )
+
+    return out
+
+
+@mpath.command(help="Print contents of file as text.")
+@options.arg_path
+def read_text(path: MPath):
+    try:
+        click.echo(path.read_text())
+    except Exception as exc:  # pragma: no cover
+        raise click.ClickException(exc)
+
+
+@mpath.command(help="Print contents of file as JSON.")
+@options.arg_path
+@click.option("--indent", "-i", type=click.INT, default=4)
+def read_json(path: MPath, indent: int = 4):
+    try:
+        click.echo(json.dumps(path.read_json(), indent=indent))
+    except Exception as exc:  # pragma: no cover
+        raise click.ClickException(exc)

mapchete/cli/options.py

@@ -130,6 +130,8 @@ def _cb_none_concurrency(ctx, param, value):
 arg_src_tiledir = click.argument("src_tiledir", type=click.STRING)
 arg_dst_tiledir = click.argument("dst_tiledir", type=click.STRING)
 arg_tiledir = click.argument("tiledir", type=click.STRING)
+arg_path = click.argument("path", type=click.Path(path_type=MPath))
+arg_out_path = click.argument("out_path", type=click.Path(path_type=MPath))
 
 
 # click options #
@@ -202,7 +204,7 @@ def _cb_none_concurrency(ctx, param, value):
     "--tile", "-t", type=click.INT, nargs=3, help="Zoom, row, column of single tile."
 )
 opt_overwrite = click.option(
-    "--overwrite", "-o", is_flag=True, help="Overwrite if tile(s) already exist(s)."
+    "--overwrite", "-o", is_flag=True, help="Overwrite if output already exist(s)."
 )
 opt_workers = click.option(
     "--workers",

mapchete/formats/default/_fiona_base.py

@@ -5,8 +5,6 @@
 import logging
 import types
 
-from fiona.errors import DriverError
-
 from mapchete.formats import base
 from mapchete.io import MPath, fiona_open
 from mapchete.io.vector import write_vector_window
@@ -57,19 +55,10 @@ def read(self, output_tile, **kwargs):
         process output : list
         """
         try:
-            path = self.get_path(output_tile)
-            with fiona_open(path, "r") as src:
+            with fiona_open(self.get_path(output_tile), "r") as src:
                 return list(src)
-        except DriverError as e:
-            for i in (
-                "does not exist in the file system",
-                "No such file or directory",
-                "specified key does not exist.",
-            ):
-                if i in str(e):
-                    return self.empty(output_tile)
-            else:  # pragma: no cover
-                raise
+        except FileNotFoundError:
+            return self.empty(output_tile)
 
     def is_valid_with_config(self, config):
         """

mapchete/io/vector.py

@@ -63,10 +63,21 @@ def fiona_read(path, mode="r", **kwargs):
     """
     path = MPath.from_inp(path)
 
-    with path.fio_env() as env:
-        logger.debug("reading %s with GDAL options %s", str(path), env.options)
-        with fiona.open(str(path), mode=mode, **kwargs) as src:
-            yield src
+    try:
+        with path.fio_env() as env:
+            logger.debug("reading %s with GDAL options %s", str(path), env.options)
+            with fiona.open(str(path), mode=mode, **kwargs) as src:
+                yield src
+    except DriverError as exc:
+        for i in (
+            "does not exist in the file system",
+            "No such file or directory",
+            "specified key does not exist.",
+        ):
+            if i in str(repr(exc)):
+                raise FileNotFoundError(f"path {str(path)} does not exist")
+        else:  # pragma: no cover
+            raise
 
 
 @contextmanager
@@ -335,35 +346,8 @@ def _get_reprojected_features(
     logger.debug("reading %s", inp)
     with ExitStack() as exit_stack:
         if isinstance(inp, (str, MPath)):
-            try:
-                src = exit_stack.enter_context(fiona_open(inp, "r"))
-                src_crs = CRS(src.crs)
-            except Exception as e:
-                # fiona errors which indicate file does not exist
-                for i in (
-                    "does not exist in the file system",
-                    "No such file or directory",
-                    "The specified key does not exist",
-                ):
-                    if i in str(e):
-                        raise FileNotFoundError(
-                            "%s not found and cannot be opened with Fiona" % inp
-                        )
-                else:
-                    try:
-                        # NOTE: this can cause addional S3 requests
-                        exists = path_exists(inp)
-                    except Exception:  # pragma: no cover
-                        # in order not to mask the original fiona exception, raise it
-                        raise e
-                    if exists:
-                        # raise fiona exception
-                        raise e
-                    else:  # pragma: no cover
-                        # file does not exist
-                        raise FileNotFoundError(
-                            "%s not found and cannot be opened with Fiona" % inp
-                        )
+            src = exit_stack.enter_context(fiona_open(inp, "r"))
+            src_crs = CRS(src.crs)
         else:
             src = inp
             src_crs = inp.crs

mapchete/log.py

@@ -7,7 +7,6 @@
 """
 import logging
 import sys
-import warnings
 from itertools import chain
 
 from mapchete.registered import drivers, processes
@@ -99,24 +98,3 @@ def setup_logfile(logfile):
     for i in all_mapchete_packages:
         logging.getLogger(i).addHandler(file_handler)
         logging.getLogger(i).setLevel(logging.DEBUG)
-
-
-def user_process_logger(pname):
-    """Logger to be used within a user process file."""
-    warnings.warn(
-        DeprecationWarning(
-            "user_process_logger() deprecated, you can use standard logging module "
-            "instead."
-        )
-    )
-    return logging.getLogger("mapchete.user_process." + pname)
-
-
-def driver_logger(dname):
-    """Logger to be used from a driver plugin."""
-    warnings.warn(
-        DeprecationWarning(
-            "driver_logger() deprecated, you can use standard logging module instead."
-        )
-    )
-    return logging.getLogger("mapchete.formats.drivers." + dname)