diff --git a/.github/workflows/ci_tests.yml b/.github/workflows/ci_tests.yml index 1fea87a..674a2b8 100644 --- a/.github/workflows/ci_tests.yml +++ b/.github/workflows/ci_tests.yml @@ -82,6 +82,7 @@ jobs: --name "entity-service" \ --network "host" \ --volume "${PWD}:/app" \ + --user "$(id -u):$(id -g)" \ entity-service sleep 5 @@ -94,7 +95,7 @@ jobs: - name: Run tests run: | { - pytest -vv --live-backend + pytest -vv --live-backend --cov-report=xml } || { echo "Failed! Here's the Docker logs for the service:" && docker logs entity-service && @@ -105,6 +106,17 @@ jobs: exit 1 } + - name: Upload coverage + if: github.repository_owner == 'SINTEF' + uses: codecov/codecov-action@v3 + with: + fail_ci_if_error: true + env_vars: OS,PYTHON + flags: docker + env: + OS: ubuntu-latest + PYTHON: '3.10' + pytest: runs-on: ubuntu-latest @@ -137,6 +149,7 @@ jobs: with: fail_ci_if_error: true env_vars: OS,PYTHON + flags: local env: OS: ubuntu-latest PYTHON: ${{ matrix.python_version }} diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 6175107..d5bbac7 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -62,7 +62,7 @@ repos: rev: v1.7.0 hooks: - id: mypy - exclude: ^docs/example/.*$$ + exclude: ^(docs/example|tests)/.*$ additional_dependencies: - pydantic>=2 - types-requests diff --git a/dlite_entities_service/backend.py b/dlite_entities_service/backend.py deleted file mode 100644 index 66c6de5..0000000 --- a/dlite_entities_service/backend.py +++ /dev/null @@ -1,25 +0,0 @@ -"""Backend implementation.""" -from __future__ import annotations - -from pymongo import MongoClient - -from dlite_entities_service.config import CONFIG - -client_kwargs = { - "username": CONFIG.mongo_user, - "password": CONFIG.mongo_password.get_secret_value() - if CONFIG.mongo_password is not None - else None, -} -for key, value in list(client_kwargs.items()): - if value is None: - client_kwargs.pop(key, None) - - -MONGO_CLIENT = MongoClient( - str(CONFIG.mongo_uri), - **client_kwargs, -) -MONGO_DB = MONGO_CLIENT.dlite - -ENTITIES_COLLECTION = MONGO_DB.entities diff --git a/dlite_entities_service/main.py b/dlite_entities_service/main.py index bc80bc2..dfcea25 100644 --- a/dlite_entities_service/main.py +++ b/dlite_entities_service/main.py @@ -9,10 +9,10 @@ from fastapi import FastAPI, HTTPException, Path, status from dlite_entities_service import __version__ -from dlite_entities_service.backend import ENTITIES_COLLECTION -from dlite_entities_service.config import CONFIG -from dlite_entities_service.logger import setup_logger from dlite_entities_service.models import VersionedSOFTEntity +from dlite_entities_service.service.backend import ENTITIES_COLLECTION +from dlite_entities_service.service.config import CONFIG +from dlite_entities_service.service.logger import setup_logger if TYPE_CHECKING: # pragma: no cover from typing import Any diff --git a/dlite_entities_service/models/soft5.py b/dlite_entities_service/models/soft5.py index 7f0d3db..1f9a18e 100644 --- a/dlite_entities_service/models/soft5.py +++ b/dlite_entities_service/models/soft5.py @@ -6,7 +6,7 @@ from pydantic import BaseModel, Field, field_validator, model_validator from pydantic.networks import AnyHttpUrl -from dlite_entities_service.config import CONFIG +from dlite_entities_service.service.config import CONFIG class SOFT5Dimension(BaseModel): diff --git a/dlite_entities_service/models/soft7.py b/dlite_entities_service/models/soft7.py index 9d7f059..d14ab44 100644 --- a/dlite_entities_service/models/soft7.py +++ b/dlite_entities_service/models/soft7.py @@ -6,7 +6,7 @@ from pydantic import BaseModel, Field, field_validator, model_validator from pydantic.networks import AnyHttpUrl -from dlite_entities_service.config import CONFIG +from dlite_entities_service.service.config import CONFIG class SOFT7Property(BaseModel): diff --git a/dlite_entities_service/service/__init__.py b/dlite_entities_service/service/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/dlite_entities_service/service/backend.py b/dlite_entities_service/service/backend.py new file mode 100644 index 0000000..14af3ea --- /dev/null +++ b/dlite_entities_service/service/backend.py @@ -0,0 +1,43 @@ +"""Backend implementation.""" +from __future__ import annotations + +from typing import TYPE_CHECKING + +from pymongo import MongoClient +from pymongo.errors import WriteConcernError, WriteError + +from dlite_entities_service.service.config import CONFIG + +if TYPE_CHECKING: # pragma: no cover + from pymongo.collection import Collection + + +AnyWriteError = (WriteError, WriteConcernError) +"""Any write error exception.""" + + +def get_collection( + uri: str | None = None, username: str | None = None, password: str | None = None +) -> Collection: + """Get the MongoDB collection for entities.""" + client_kwargs = { + "username": username or CONFIG.mongo_user, + "password": password + or ( + CONFIG.mongo_password.get_secret_value() + if CONFIG.mongo_password is not None + else None + ), + } + for key, value in list(client_kwargs.items()): + if value is None: + client_kwargs.pop(key, None) + + mongo_client = MongoClient( + uri or str(CONFIG.mongo_uri), + **client_kwargs, + ) + return mongo_client.dlite.entities + + +ENTITIES_COLLECTION = get_collection() diff --git a/dlite_entities_service/config.py b/dlite_entities_service/service/config.py similarity index 100% rename from dlite_entities_service/config.py rename to dlite_entities_service/service/config.py diff --git a/dlite_entities_service/logger.py b/dlite_entities_service/service/logger.py similarity index 96% rename from dlite_entities_service/logger.py rename to dlite_entities_service/service/logger.py index 6c9e2b1..3260265 100644 --- a/dlite_entities_service/logger.py +++ b/dlite_entities_service/service/logger.py @@ -44,7 +44,7 @@ def disable_logging(): def _get_service_logger_handlers() -> list[logging.Handler]: """Return a list of handlers for the service logger.""" # Create logs directory - root_dir = Path(__file__).resolve().parent.parent.resolve() + root_dir = Path(__file__).resolve().parent.parent.parent.resolve() logs_dir = root_dir / "logs" logs_dir.mkdir(exist_ok=True) diff --git a/dlite_entities_service/utils_cli/__init__.py b/dlite_entities_service/utils_cli/__init__.py new file mode 100644 index 0000000..de7dcfd --- /dev/null +++ b/dlite_entities_service/utils_cli/__init__.py @@ -0,0 +1,5 @@ +"""Utility CLI + +This module contains a CLI with utilities that may be useful when dealing with the +DLite entities service, it's data (backend), and possibly more. +""" diff --git a/dlite_entities_service/utils_cli/_utils/__init__.py b/dlite_entities_service/utils_cli/_utils/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/dlite_entities_service/utils_cli/_utils/global_settings.py b/dlite_entities_service/utils_cli/_utils/global_settings.py new file mode 100644 index 0000000..350e20b --- /dev/null +++ b/dlite_entities_service/utils_cli/_utils/global_settings.py @@ -0,0 +1,92 @@ +"""Global settings for the CLI.""" +from __future__ import annotations + +from pathlib import Path +from typing import Optional + +try: + import typer +except ImportError as exc: + raise ImportError( + "Please install the DLite entities service utility CLI with 'pip install " + f"{Path(__file__).resolve().parent.parent.parent.parent.resolve()}[cli]'" + ) from exc + +from rich import print # pylint: disable=redefined-builtin + +from dlite_entities_service import __version__ + +STATUS = {"use_service_dotenv": False} + +# Type Aliases +OptionalBool = Optional[bool] + + +def print_version(value: bool) -> None: + """Print version and exit.""" + if value: + print(f"dlite-entities-service version: {__version__}") + raise typer.Exit() + + +def global_options( + _: OptionalBool = typer.Option( + None, + "--version", + help="Show version and exit", + is_eager=True, + callback=print_version, + ), + use_service_dotenv: bool = typer.Option( + False, + "--use-service-dotenv/--use-cli-dotenv", + help=( + "Use the .env file also used for the DLite Entities Service or one " + "only for the CLI." + ), + is_flag=True, + rich_help_panel="Global options", + ), + as_json: bool = typer.Option( + False, + "--json", + help=( + "Print output as JSON. (Muting mutually exclusive with --yaml/--yml " + "and --json-one-line.)" + ), + is_flag=True, + rich_help_panel="Global options", + ), + as_json_one_line: bool = typer.Option( + False, + "--json-one-line", + help=( + "Print output as JSON without new lines. (Muting mutually exclusive " + "with --yaml/--yml and --json.)" + ), + is_flag=True, + rich_help_panel="Global options", + ), + as_yaml: bool = typer.Option( + False, + "--yaml", + "--yml", + help=( + "Print output as YAML. (Mutually exclusive with --json and " + "--json-one-line.)" + ), + is_flag=True, + rich_help_panel="Global options", + ), +) -> None: + """Global options for the CLI.""" + STATUS["use_service_dotenv"] = use_service_dotenv + + if sum(int(_) for _ in [as_json, as_json_one_line, as_yaml]) > 1: + raise typer.BadParameter( + "Cannot use --json, --yaml/--yml, and --json-one-line together in any " + "combination." + ) + STATUS["as_json"] = as_json + STATUS["as_json_one_line"] = as_json_one_line + STATUS["as_yaml"] = as_yaml diff --git a/dlite_entities_service/utils_cli/config.py b/dlite_entities_service/utils_cli/config.py new file mode 100644 index 0000000..0055c16 --- /dev/null +++ b/dlite_entities_service/utils_cli/config.py @@ -0,0 +1,207 @@ +"""config subcommand for dlite-entities-service CLI.""" +# pylint: disable=duplicate-code +from __future__ import annotations + +import sys +from collections.abc import Generator +from pathlib import Path +from typing import Optional + +if sys.version_info >= (3, 11): + from enum import StrEnum +else: + from enum import Enum + + class StrEnum(str, Enum): + """Enum with string values.""" + + +try: + import typer +except ImportError as exc: + raise ImportError( + "Please install the DLite entities service utility CLI with " + f"'pip install {Path(__file__).resolve().parent.parent.parent.resolve()}[cli]'" + ) from exc + +import yaml +from dotenv import dotenv_values, set_key, unset_key +from rich import print, print_json # pylint: disable=redefined-builtin +from rich.console import Console + +from dlite_entities_service.service.config import CONFIG +from dlite_entities_service.utils_cli._utils.global_settings import STATUS + +ERROR_CONSOLE = Console(stderr=True) +CLI_DOTENV_FILE: Path = ( + Path(__file__).resolve().parent / CONFIG.model_config["env_file"] +) +SERVICE_DOTENV_FILE: Path = ( + Path(__file__).resolve().parent.parent.parent / CONFIG.model_config["env_file"] +) + +APP = typer.Typer( + name=__file__.rsplit("/", 1)[-1].replace(".py", ""), + help="Manage configuration options.", + no_args_is_help=True, + invoke_without_command=True, +) + +# Type Aliases +OptionalStr = Optional[str] + + +class ConfigFields(StrEnum): + """Configuration options.""" + + BASE_URL = "base_url" + MONGO_URI = "mongo_uri" + MONGO_USER = "mongo_user" + MONGO_PASSWORD = "mongo_password" # nosec + + @classmethod + def autocomplete(cls, incomplete: str) -> Generator[tuple[str, str], None, None]: + """Return a list of valid configuration options.""" + for member in cls: + if member.value.startswith(incomplete): + if member.value not in CONFIG.model_fields: + raise typer.BadParameter( + f"Invalid configuration option: {member.value!r}" + ) + yield member.value, CONFIG.model_fields[member.value].description + + def is_sensitive(self) -> bool: + """Return True if this is a sensitive configuration option.""" + return self in [ConfigFields.MONGO_PASSWORD] + + +@APP.command(name="set") +def set_config( + key: ConfigFields = typer.Argument( + help=( + "Configuration option to set. These can also be set as an environment " + f"variable by prefixing with {CONFIG.model_config['env_prefix']!r}." + ), + show_choices=True, + # Start using shell_complete once tiangolo/typer#334 is resolved. + # shell_complete=ConfigFields.autocomplete, + autocompletion=ConfigFields.autocomplete, + case_sensitive=False, + show_default=False, + ), + value: OptionalStr = typer.Argument( + None, + help=( + "Value to set. For sensitive values, this will be prompted for if not " + "provided." + ), + show_default=False, + ), +) -> None: + """Set a configuration option.""" + if not value: + value = typer.prompt(f"Enter value for {key}", hide_input=key.is_sensitive()) + if STATUS["use_service_dotenv"]: + dotenv_file = SERVICE_DOTENV_FILE + else: + dotenv_file = CLI_DOTENV_FILE + if not dotenv_file.exists(): + dotenv_file.touch() + set_key(dotenv_file, f"{CONFIG.model_config['env_prefix']}{key}", value) + print( + f"Set {CONFIG.model_config['env_prefix']}{key} to sensitive value." + if key.is_sensitive() + else f"Set {CONFIG.model_config['env_prefix']}{key} to {value}." + ) + + +@APP.command() +def unset( + key: ConfigFields = typer.Argument( + None, + help="Configuration option to unset.", + show_choices=True, + # Start using shell_complete once tiangolo/typer#334 is resolved. + # shell_complete=ConfigFields.autocomplete, + autocompletion=ConfigFields.autocomplete, + case_sensitive=False, + show_default=False, + ), +) -> None: + """Unset a single configuration option.""" + if STATUS["use_service_dotenv"]: + dotenv_file = SERVICE_DOTENV_FILE + else: + dotenv_file = CLI_DOTENV_FILE + if dotenv_file.exists(): + unset_key(dotenv_file, f"{CONFIG.model_config['env_prefix']}{key}") + print(f"Unset {CONFIG.model_config['env_prefix']}{key}.") + + +@APP.command() +def unset_all() -> None: + """Unset all configuration options.""" + typer.confirm( + "Are you sure you want to unset (remove) all configuration options in " + f"{'Service' if STATUS['use_service_dotenv'] else 'CLI'}-specific " + f"{CONFIG.model_config['env_file']} file?", + abort=True, + ) + + if STATUS["use_service_dotenv"]: + dotenv_file = SERVICE_DOTENV_FILE + else: + dotenv_file = CLI_DOTENV_FILE + if dotenv_file.exists(): + dotenv_file.unlink() + print(f"Unset all configuration options. (Removed {dotenv_file}.)") + else: + print(f"Unset all configuration options. ({dotenv_file} file not found.)") + + +@APP.command() +def show( + reveal_sensitive: bool = typer.Option( + False, + "--reveal-sensitive", + help="Reveal sensitive values. (DANGEROUS! Use with caution.)", + is_flag=True, + show_default=False, + ), +) -> None: + """Show the current configuration.""" + if STATUS["use_service_dotenv"]: + dotenv_file = SERVICE_DOTENV_FILE + else: + dotenv_file = CLI_DOTENV_FILE + if dotenv_file.exists(): + if not any(STATUS[_] for _ in ["as_json", "as_json_one_line", "as_yaml"]): + print(f"Current configuration in {dotenv_file}:\n") + values = { + ConfigFields(key[len(CONFIG.model_config["env_prefix"]) :]): value + for key, value in dotenv_values(dotenv_file).items() + if key + in [ + f"{CONFIG.model_config['env_prefix']}{_}" + for _ in ConfigFields.__members__.values() + ] + } + else: + ERROR_CONSOLE.print(f"No {dotenv_file} file found.") + raise typer.Exit(1) + + output = {} + for key, value in values.items(): + sensitive_value = None + if not reveal_sensitive and key.is_sensitive(): + sensitive_value = "*" * 8 + output[f"{CONFIG.model_config['env_prefix']}{key}"] = sensitive_value or value + + if STATUS["as_json"] or STATUS["as_json_one_line"]: + print_json(data=output, indent=2 if STATUS["as_json"] else None) + elif STATUS["as_yaml"]: + print(yaml.safe_dump(output, sort_keys=False, allow_unicode=True)) + else: + print( + "\n".join(f"[bold]{key}[/bold]: {value}" for key, value in output.items()) + ) diff --git a/dlite_entities_service/utils_cli/main.py b/dlite_entities_service/utils_cli/main.py new file mode 100644 index 0000000..f649951 --- /dev/null +++ b/dlite_entities_service/utils_cli/main.py @@ -0,0 +1,557 @@ +"""Typer CLI for doing DLite entities service stuff.""" +# pylint: disable=duplicate-code +from __future__ import annotations + +import json +import os +import sys +from pathlib import Path +from typing import TYPE_CHECKING, Optional + +if sys.version_info >= (3, 11): + from enum import StrEnum +else: + from enum import Enum + + class StrEnum(str, Enum): + """Enum with string values.""" + + +try: + import typer +except ImportError as exc: + raise ImportError( + "Please install the DLite entities service utility CLI with " + f"'pip install {Path(__file__).resolve().parent.parent.parent.resolve()}[cli]'" + ) from exc + +import dlite +import yaml +from dotenv import dotenv_values, find_dotenv +from rich import print # pylint: disable=redefined-builtin +from rich.console import Console + +from dlite_entities_service.service.backend import ( + ENTITIES_COLLECTION, + AnyWriteError, + get_collection, +) +from dlite_entities_service.utils_cli._utils.global_settings import global_options +from dlite_entities_service.utils_cli.config import APP as config_APP + +if TYPE_CHECKING: # pragma: no cover + from typing import Any + + from pymongo.collection import Collection + + +ERROR_CONSOLE = Console(stderr=True) + + +class EntityFileFormats(StrEnum): + """Supported entity file formats.""" + + JSON = "json" + YAML = "yaml" + YML = "yml" + + +# Type Aliases +OptionalListEntityFileFormats = Optional[list[EntityFileFormats]] +OptionalListStr = Optional[list[str]] +OptionalListPath = Optional[list[Path]] +OptionalStr = Optional[str] + + +APP = typer.Typer( + name="entities-service", + help="DLite entities service utility CLI", + no_args_is_help=True, + pretty_exceptions_show_locals=False, + callback=global_options, +) +APP.add_typer(config_APP, callback=global_options) + + +def _get_backend() -> Collection: + """Return the backend.""" + config_file = find_dotenv() + if config_file: + config = dotenv_values(config_file) + backend_options = { + "uri": config.get("entity_service_mongo_uri"), + "username": config.get("entity_service_mongo_user"), + "password": config.get("entity_service_mongo_password"), + } + if all(_ is None for _ in backend_options.values()): + return ENTITIES_COLLECTION + return get_collection(**backend_options) + return ENTITIES_COLLECTION + + +@APP.command(no_args_is_help=True) +def upload( + filepaths: OptionalListPath = typer.Option( + None, + "--file", + "-f", + exists=True, + file_okay=True, + dir_okay=False, + readable=True, + resolve_path=True, + help="Path to DLite entity file.", + show_default=False, + ), + directories: OptionalListPath = typer.Option( + None, + "--dir", + "-d", + exists=True, + file_okay=False, + dir_okay=True, + readable=True, + resolve_path=True, + help=( + "Path to directory with DLite entities. All files matching the given " + "format(s) in the directory will be uploaded. " + "Subdirectories will be ignored." + ), + show_default=False, + ), + file_formats: OptionalListEntityFileFormats = typer.Option( + [EntityFileFormats.JSON], + "--format", + help="Format of DLite entity file(s).", + show_choices=True, + show_default=True, + case_sensitive=False, + ), +) -> None: + """Upload (local) DLite entities to a remote location.""" + unique_filepaths = set(filepaths or []) + directories = list(set(directories or [])) + file_formats = list(set(file_formats or [])) + + if not filepaths and not directories: + ERROR_CONSOLE.print( + "[bold red]Error[/bold red]: Missing either option '--file' / '-f' or " + "'--dir' / '-d'." + ) + raise typer.Exit(1) + + for directory in directories: + for root, _, files in os.walk(directory): + unique_filepaths |= { + Path(root) / file + for file in files + if file.lower().endswith(tuple(file_formats)) + } + + if not unique_filepaths: + ERROR_CONSOLE.print( + "[bold red]Error[/bold red]: No files found with the given options." + ) + raise typer.Exit(1) + + successes = [] + for filepath in unique_filepaths: + if filepath.suffix[1:].lower() not in file_formats: + ERROR_CONSOLE.print( + "[bold yellow]Warning[/bold yellow]: File format " + f"{filepath.suffix[1:].lower()!r} is not supported. Skipping file: " + f"{filepath}" + ) + continue + + entity: dict[str, Any] = ( + json.loads(filepath.read_bytes()) + if filepath.suffix[1:].lower() == "json" + else yaml.safe_load(filepath.read_bytes()) + ) + + try: + dlite.Instance.from_dict(entity, single=True, check_storages=False) + except ( # pylint: disable=redefined-outer-name + dlite.DLiteError, + KeyError, + ) as exc: + ERROR_CONSOLE.print( + f"[bold red]Error[/bold red]: {filepath} cannot be loaded with DLite. " + f"DLite exception: {exc}" + ) + raise typer.Exit(1) from exc + + try: + _get_backend().insert_one(entity) + except AnyWriteError as exc: + ERROR_CONSOLE.print( + f"[bold red]Error[/bold red]: {filepath} cannot be uploaded. " + f"Backend exception: {exc}" + ) + raise typer.Exit(1) from exc + + successes.append(filepath) + + if successes: + print( + f"Successfully uploaded {len(successes)} entities: " + f"{[str(_) for _ in successes]}" + ) + else: + print("No entities were uploaded.") + + +@APP.command(hidden=True) +def iterate() -> None: + """Iterate on an existing DLite entity. + + This means uploading a new version of an existing entity. + """ + print("Not implemented yet") + + +@APP.command(no_args_is_help=True) +def update( + filepaths: OptionalListPath = typer.Option( + None, + "--file", + "-f", + exists=True, + file_okay=True, + dir_okay=False, + readable=True, + resolve_path=True, + help="Path to DLite entity file.", + show_default=False, + ), + directories: OptionalListPath = typer.Option( + None, + "--dir", + "-d", + exists=True, + file_okay=False, + dir_okay=True, + readable=True, + resolve_path=True, + help=( + "Path to directory with DLite entities. All files matching the given " + "format(s) in the directory will be uploaded. " + "Subdirectories will be ignored." + ), + show_default=False, + ), + file_formats: OptionalListEntityFileFormats = typer.Option( + [EntityFileFormats.JSON], + "--format", + help="Format of DLite entity file(s).", + show_choices=True, + show_default=True, + case_sensitive=False, + ), + insert: bool = typer.Option( + False, + "--insert", + "-i", + help="Insert the entity if it does not exist yet.", + show_default=False, + is_flag=True, + ), +) -> None: + """Update an existing (remote) DLite entity.""" + unique_filepaths = set(filepaths or []) + directories = list(set(directories or [])) + file_formats = list(set(file_formats or [])) + + if not filepaths and not directories: + ERROR_CONSOLE.print( + "[bold red]Error[/bold red]: Missing either option '--file' / '-f' or " + "'--dir' / '-d'." + ) + raise typer.Exit(1) + + for directory in directories: + for root, _, files in os.walk(directory): + unique_filepaths |= { + Path(root) / file + for file in files + if file.lower().endswith(tuple(file_formats)) + } + + if not unique_filepaths: + ERROR_CONSOLE.print( + "[bold red]Error[/bold red]: No files found with the given options." + ) + raise typer.Exit(1) + + successes = [] + inserted = [] + for filepath in unique_filepaths: + if filepath.suffix[1:].lower() not in file_formats: + ERROR_CONSOLE.print( + "[bold yellow]Warning[/bold yellow]: File format " + f"{filepath.suffix[1:].lower()!r} is not supported. Skipping file: " + f"{filepath}" + ) + continue + + entity: dict[str, Any] = ( + json.loads(filepath.read_bytes()) + if filepath.suffix[1:].lower() == "json" + else yaml.safe_load(filepath.read_bytes()) + ) + + try: + dlite.Instance.from_dict(entity, single=True, check_storages=False) + except ( # pylint: disable=redefined-outer-name + dlite.DLiteError, + KeyError, + ) as exc: + ERROR_CONSOLE.print( + f"[bold red]Error[/bold red]: {filepath} cannot be loaded with DLite. " + f"DLite exception: {exc}" + ) + raise typer.Exit(1) from exc + + try: + result = _get_backend().update_one( + filter={"uri": entity["uri"]}, + update=entity, + upsert=insert, + ) + except AnyWriteError as exc: + ERROR_CONSOLE.print( + f"[bold red]Error[/bold red]: {filepath} cannot be uploaded. " + f"Backend exception: {exc}" + ) + raise typer.Exit(1) from exc + + if insert and result.upserted_id: + inserted.append(filepath) + + successes.append(filepath) + + if successes and inserted: + print( + f"Successfully updated {len(successes) - len(inserted)} entities and " + f"inserted {len(inserted)} new entities: " + f"{[str(_) for _ in successes if _ not in inserted]} " + f"and {[str(_) for _ in inserted]}" + ) + elif successes: + print( + f"Successfully updated {len(successes)} entities: " + f"{[str(_) for _ in successes]}" + ) + else: + print("No entities were updated.") + + +@APP.command(no_args_is_help=True) +def delete( + uri: str = typer.Argument( + help="URI of the DLite entity to delete.", + show_default=False, + ), +) -> None: + """Delete an existing (remote) DLite entity.""" + backend = _get_backend() + + if not backend.count_documents({"uri": uri}): + print(f"Already no entity found with URI {uri!r}.") + raise typer.Exit() + + typer.confirm( + f"Are you sure you want to delete entity with URI {uri!r}?", abort=True + ) + + backend.delete_one({"uri": uri}) + if backend.count_documents({"uri": uri}): + ERROR_CONSOLE.print( + f"[bold red]Error[/bold red]: Failed to delete entity with URI {uri!r}." + ) + raise typer.Exit(1) + print(f"Successfully deleted entity with URI {uri!r}.") + + +@APP.command(no_args_is_help=True) +def get( + uri: str = typer.Argument( + help="URI of the DLite entity to get.", + show_default=False, + ), +) -> None: + """Get an existing (remote) DLite entity.""" + backend = _get_backend() + + if not backend.count_documents({"uri": uri}): + ERROR_CONSOLE.print( + f"[bold red]Error[/bold red]: No entity found with URI {uri!r}." + ) + raise typer.Exit(1) + + entity_dict: dict[str, Any] = backend.find_one({"uri": uri}) + entity_dict.pop("_id") + entity = dlite.Instance.from_dict(entity_dict, single=True, check_storages=False) + print(entity) + + +@APP.command(no_args_is_help=True) +def search( + uris: OptionalListStr = typer.Argument( + None, + metavar="[URI]...", + help=( + "URI of the DLite entity to search for. Multiple URIs can be provided. " + "Note, the 'http://onto-ns.com/meta' prefix is optional." + ), + show_default=False, + ), + query: OptionalStr = typer.Option( + None, + "--query", + "-q", + help="Backend-specific query to search for DLite entities.", + show_default=False, + ), + as_json: bool = typer.Option( + False, + "--json", + "-j", + help="Return the search results as JSON.", + show_default=False, + is_flag=True, + ), +) -> None: + """Search for (remote) DLite entities.""" + backend = _get_backend() + + if not uris and not query: + ERROR_CONSOLE.print( + "[bold red]Error[/bold red]: Missing either argument 'URI' or option " + "'query'." + ) + raise typer.Exit(1) + + backend_query: dict[str, Any] | None = json.loads(query) if query else None + if uris: + uris = [ + uri + if uri.startswith("http://onto-ns.com/meta") + else f"http://onto-ns.com/meta/{uri.lstrip('/')}" + for uri in uris + ] + backend_query = ( + {"$and": [{"uri": {"$in": uris}}, backend_query]} + if backend_query + else {"uri": {"$in": uris}} + ) + + if backend_query is None: + ERROR_CONSOLE.print("[bold red]Error[/bold red]: Internal CLI error.") + raise typer.Exit(1) + + found_entities: list[dlite.Instance] = [] + for raw_entity in backend.find(backend_query): + raw_entity.pop("_id") + entity: dlite.Instance = dlite.Instance.from_dict( + raw_entity, single=True, check_storages=False + ) + found_entities.append(entity) + + if as_json: + print(json.dumps([_.asdict(uuid=False) for _ in found_entities])) + raise typer.Exit() + + print(f"Found {len(found_entities)} entities: {[_.uuid for _ in found_entities]}") + + +@APP.command(no_args_is_help=True) +def validate( + filepaths: OptionalListPath = typer.Option( + None, + "--file", + "-f", + exists=True, + file_okay=True, + dir_okay=False, + readable=True, + resolve_path=True, + help="Path to DLite entity file.", + show_default=False, + ), + directories: OptionalListPath = typer.Option( + None, + "--dir", + "-d", + exists=True, + file_okay=False, + dir_okay=True, + readable=True, + resolve_path=True, + help=( + "Path to directory with DLite entities. All files matching the given " + "format(s) in the directory will be validated. " + "Subdirectories will be ignored." + ), + show_default=False, + ), + file_formats: OptionalListEntityFileFormats = typer.Option( + [EntityFileFormats.JSON], + "--format", + help="Format of DLite entity file(s).", + show_choices=True, + show_default=True, + case_sensitive=False, + ), +) -> None: + """Validate (local) DLite entities.""" + unique_filepaths = set(filepaths or []) + directories = list(set(directories or [])) + file_formats = list(set(file_formats or [])) + + if not filepaths and not directories: + ERROR_CONSOLE.print( + "[bold red]Error[/bold red]: Missing either option '--file' / '-f' or " + "'--dir' / '-d'." + ) + raise typer.Exit(1) + + for directory in directories: + for root, _, files in os.walk(directory): + unique_filepaths |= { + Path(root) / file + for file in files + if file.lower().endswith(tuple(file_formats)) + } + + if not unique_filepaths: + ERROR_CONSOLE.print( + "[bold red]Error[/bold red]: No files found with the given options." + ) + raise typer.Exit(1) + + for filepath in unique_filepaths: + if filepath.suffix[1:].lower() not in file_formats: + ERROR_CONSOLE.print( + "[bold yellow]Warning[/bold yellow]: File format " + f"{filepath.suffix[1:].lower()!r} is not supported. Skipping file: " + f"{filepath}" + ) + continue + + entity: dict[str, Any] = ( + json.loads(filepath.read_bytes()) + if filepath.suffix[1:].lower() == "json" + else yaml.safe_load(filepath.read_bytes()) + ) + + try: + dlite.Instance.from_dict(entity, single=True, check_storages=False) + except ( # pylint: disable=redefined-outer-name + dlite.DLiteError, + KeyError, + ): + print(f"{filepath} [bold red]invalid[/bold red]") + else: + print(f"{filepath} [bold green]valid[/bold green]") diff --git a/pyproject.toml b/pyproject.toml index 0fbd1d2..47a31e6 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -35,19 +35,26 @@ dependencies = [ ] [project.optional-dependencies] -testing = [ +cli = [ "dlite-python >=0.4.5,<1; python_version < '3.12'", + "typer[all] >=0.9.0,<1", +] +testing = [ "httpx >=0.25.1,<1", "mongomock ~=4.1", "pytest ~=7.4", "pytest-cov ~=4.1", "pyyaml ~=6.0", + "dlite-entities-service[cli]", ] dev = [ "pre-commit ~=3.5", - "dlite-entities-service[testing]", + "dlite-entities-service[testing,cli]", ] +[project.scripts] +entities-service = "dlite_entities_service.utils_cli.main:APP" + [project.urls] Home = "https://github.com/CasperWA/dlite-entities-service" Documentation = "https://CasperWA.github.io/dlite-entities-service" @@ -67,42 +74,44 @@ plugins = ["pydantic.mypy"] [tool.ruff.lint] extend-select = [ - "E", # pycodestyle - "F", # pyflakes - "B", # flake8-bugbear - "BLE", # flake8-blind-except - "I", # isort - "ARG", # flake8-unused-arguments - "C4", # flake8-comprehensions - "EM", # flake8-errmsg - "ICN", # flake8-import-conventions - "G", # flake8-logging-format - "PGH", # pygrep-hooks - "PIE", # flake8-pie - "PL", # pylint - "PT", # flake8-pytest-style - "PTH", # flake8-use-pathlib - "RET", # flake8-return - "RUF", # Ruff-specific - "SIM", # flake8-simplify - "T20", # flake8-print - "YTT", # flake8-2020 - "EXE", # flake8-executable - "NPY", # NumPy specific rules - "PD", # pandas-vet - "PYI", # flake8-pyi + "E", # pycodestyle + "F", # pyflakes + "B", # flake8-bugbear + "BLE", # flake8-blind-except + "I", # isort + "ARG", # flake8-unused-arguments + "C4", # flake8-comprehensions + "ICN", # flake8-import-conventions + "G", # flake8-logging-format + "PGH", # pygrep-hooks + "PIE", # flake8-pie + "PL", # pylint + "PT", # flake8-pytest-style + "PTH", # flake8-use-pathlib + "RET", # flake8-return + "RUF", # Ruff-specific + "SIM", # flake8-simplify + "T20", # flake8-print + "YTT", # flake8-2020 + "EXE", # flake8-executable + "PYI", # flake8-pyi ] ignore = [ - "PLR", # Design related pylint codes + "PLR", # Design related pylint codes + "B008", # Performing function calls in argument defaults - done all the time in the CLI. ] isort.required-imports = ["from __future__ import annotations"] [tool.pytest.ini_options] +minversion = "7.4" addopts = "-rs --cov=dlite_entities_service --cov-report=term-missing:skip-covered --no-cov-on-fail" filterwarnings = [ # Treat all warnings as errors "error", + # Remove warning filter once tiangolo/typer#334 is resolved. + "ignore:.*'autocompletion' is renamed to 'shell_complete'.*:DeprecationWarning", + # mongomock uses pkg_resources "ignore:.*pkg_resources is deprecated as an API.*:DeprecationWarning", ] diff --git a/tests/conftest.py b/tests/conftest.py index 80f69d7..1934e1f 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1,4 +1,4 @@ -"""Pytest configuration file.""" +"""Configuration and fixtures for all pytest tests.""" from __future__ import annotations from typing import TYPE_CHECKING @@ -59,7 +59,7 @@ def mongo_test_collection(static_dir: Path, live_backend: bool) -> Collection | import yaml if live_backend: - from dlite_entities_service.backend import ENTITIES_COLLECTION + from dlite_entities_service.service.backend import ENTITIES_COLLECTION # TODO: Handle authentication properly ENTITIES_COLLECTION.insert_many( @@ -71,7 +71,7 @@ def mongo_test_collection(static_dir: Path, live_backend: bool) -> Collection | # else from mongomock import MongoClient - from dlite_entities_service.config import CONFIG + from dlite_entities_service.service.config import CONFIG client_kwargs = { "username": CONFIG.mongo_user, @@ -101,7 +101,7 @@ def _mock_backend_entities_collection( if mongo_test_collection is None: return - from dlite_entities_service import backend + from dlite_entities_service.service import backend monkeypatch.setattr(backend, "ENTITIES_COLLECTION", mongo_test_collection) @@ -113,8 +113,8 @@ def client(live_backend: bool) -> TestClient: from fastapi.testclient import TestClient - from dlite_entities_service.config import CONFIG from dlite_entities_service.main import APP + from dlite_entities_service.service.config import CONFIG if live_backend: host, port = os.getenv("ENTITY_SERVICE_HOST", "localhost"), os.getenv( diff --git a/tests/static/entities.yaml b/tests/static/entities.yaml index fde7d07..f705b15 100644 --- a/tests/static/entities.yaml +++ b/tests/static/entities.yaml @@ -17,7 +17,7 @@ type: string - description: A cat. - uri: http://onto-ns.com/meta/0.1/Cat + uri: http://onto-ns.com/meta/0.2/Cat meta: http://onto-ns.com/meta/0.3/EntitySchema dimensions: ncolors: Number of different colors the cat has. @@ -40,7 +40,7 @@ # SOFT5 example - description: A dog. namespace: http://onto-ns.com/meta - version: '0.1' + version: '0.2' name: Dog dimensions: - description: Number of different colors the dog has. diff --git a/tests/static/invalid_entities/Person.json b/tests/static/invalid_entities/Person.json new file mode 100644 index 0000000..0ea09b7 --- /dev/null +++ b/tests/static/invalid_entities/Person.json @@ -0,0 +1,20 @@ +{ + "uri": "http://onto-ns.com/meta/0.1/Person", + "meta": "http://onto-ns.com/meta/0.3/EntitySchema", + "description": "A person, living or dead", + "properties": { + "skills": { + "type": "string", + "shape": ["n_skills"], + "description": "Skills of the person." + }, + "name": { + "type": "string", + "description": "Name of the person." + }, + "age": { + "type": "int", + "description": "Age of the person." + } + } +} diff --git a/tests/static/valid_entities/Cat.json b/tests/static/valid_entities/Cat.json new file mode 100644 index 0000000..89ccb17 --- /dev/null +++ b/tests/static/valid_entities/Cat.json @@ -0,0 +1,20 @@ +{ + "uri": "http://onto-ns.com/meta/0.1/Cat", + "meta": "http://onto-ns.com/meta/0.3/EntitySchema", + "description": "A cat.", + "dimensions": [], + "properties": { + "name": { + "type": "string", + "description": "Name of the cat." + }, + "age": { + "type": "int", + "description": "Age of the cat." + }, + "color": { + "type": "string", + "description": "Color of the cat." + } + } +} diff --git a/tests/static/valid_entities/Dog.json b/tests/static/valid_entities/Dog.json new file mode 100644 index 0000000..85420a7 --- /dev/null +++ b/tests/static/valid_entities/Dog.json @@ -0,0 +1,25 @@ +{ + "uri": "http://onto-ns.com/meta/0.1/Dog", + "meta": "http://onto-ns.com/meta/0.3/EntitySchema", + "description": "A good dog.", + "dimensions": {"n_tricks": "Number of tricks."}, + "properties": { + "tricks": { + "type": "string", + "shape": ["n_tricks"], + "description": "Tricks the dog can do." + }, + "name": { + "type": "string", + "description": "Name of the dog." + }, + "age": { + "type": "int", + "description": "Age of the dog." + }, + "breed": { + "type": "string", + "description": "Breed of the dog." + } + } +} diff --git a/tests/static/valid_entities/Person.json b/tests/static/valid_entities/Person.json new file mode 100644 index 0000000..1e9990e --- /dev/null +++ b/tests/static/valid_entities/Person.json @@ -0,0 +1,21 @@ +{ + "uri": "http://onto-ns.com/meta/0.1/Person", + "meta": "http://onto-ns.com/meta/0.3/EntitySchema", + "description": "A person, living or dead", + "dimensions": {"n_skills": "Number of skills."}, + "properties": { + "skills": { + "type": "string", + "shape": ["n_skills"], + "description": "Skills of the person." + }, + "name": { + "type": "string", + "description": "Name of the person." + }, + "age": { + "type": "int", + "description": "Age of the person." + } + } +} diff --git a/tests/test_route.py b/tests/test_route.py index 13cdcdc..5d65bd6 100644 --- a/tests/test_route.py +++ b/tests/test_route.py @@ -28,7 +28,8 @@ def test_get_entity( """Test the route to retrieve a DLite/SOFT entity.""" from fastapi import status - response = client.get(f"/{version}/{name}", timeout=5) + with client as client: + response = client.get(f"/{version}/{name}", timeout=5) assert ( response.is_success @@ -54,7 +55,8 @@ def test_get_entity_instance( """Validate that we can instantiate an Instance from the response""" from dlite import Instance - response = client.get(f"/{version}/{name}", timeout=5) + with client as client: + response = client.get(f"/{version}/{name}", timeout=5) assert (resolve_entity := response.json()) == entity, resolve_entity @@ -66,7 +68,8 @@ def test_get_entity_not_found(client: TestClient) -> None: from fastapi import status version, name = "0.0", "NonExistantEntity" - response = client.get(f"/{version}/{name}", timeout=5) + with client as client: + response = client.get(f"/{version}/{name}", timeout=5) assert not response.is_success, "Non existant (valid) URI returned an OK response!" assert ( @@ -83,7 +86,8 @@ def test_get_entity_invalid_uri(client: TestClient) -> None: from fastapi import status version, name = "1.0", "EntityName" - response = client.get(f"/{name}/{version}", timeout=5) + with client as client: + response = client.get(f"/{name}/{version}", timeout=5) assert not response.is_success, "Invalid URI returned an OK response!" assert ( diff --git a/tests/utils.py b/tests/utils.py index db01e23..3f122c1 100644 --- a/tests/utils.py +++ b/tests/utils.py @@ -19,7 +19,7 @@ def get_version_name(uri: str) -> tuple[str, str]: """Return the version and name part of a uri.""" import re - from dlite_entities_service.config import CONFIG + from dlite_entities_service.service.config import CONFIG namespace = str(CONFIG.base_url).rstrip("/") diff --git a/tests/utils_cli/conftest.py b/tests/utils_cli/conftest.py new file mode 100644 index 0000000..4529afa --- /dev/null +++ b/tests/utils_cli/conftest.py @@ -0,0 +1,39 @@ +"""Fixtures for the utils_cli tests.""" +from __future__ import annotations + +from typing import TYPE_CHECKING + +import pytest + +if TYPE_CHECKING: + from pymongo.collection import Collection + from typer.testing import CliRunner + + +@pytest.fixture(scope="session") +def cli() -> CliRunner: + """Fixture for CLI runner.""" + from typer.testing import CliRunner + + return CliRunner(mix_stderr=False) + + +@pytest.fixture() +def mock_entities_collection(monkeypatch: pytest.MonkeyPatch) -> Collection: + """Return a mock entities collection.""" + from mongomock import MongoClient + + from dlite_entities_service.service.config import CONFIG + from dlite_entities_service.utils_cli import main + + mongo_client = MongoClient(str(CONFIG.mongo_uri)) + mock_entities_collection = mongo_client["dlite"]["entities"] + + monkeypatch.setattr(main, "ENTITIES_COLLECTION", mock_entities_collection) + monkeypatch.setattr( + main, + "get_collection", + lambda *args, **kwargs: mock_entities_collection, # noqa: ARG005 + ) + + return mock_entities_collection diff --git a/tests/utils_cli/test_upload.py b/tests/utils_cli/test_upload.py new file mode 100644 index 0000000..37b9564 --- /dev/null +++ b/tests/utils_cli/test_upload.py @@ -0,0 +1,111 @@ +"""Tests for `entities-service upload` CLI command.""" +from __future__ import annotations + +import sys +from typing import TYPE_CHECKING + +import pytest + +if TYPE_CHECKING: + from pathlib import Path + from typing import Any + + from pymongo.collection import Collection + from typer.testing import CliRunner + + +pytestmark = pytest.mark.skipif( + sys.version_info >= (3, 12), reason="DLite does not yet support Python 3.12+." +) + + +def test_upload_no_args(cli: CliRunner) -> None: + """Test `entities-service upload` CLI command.""" + from dlite_entities_service.utils_cli.main import APP, upload + + result = cli.invoke(APP, "upload") + assert result.exit_code == 0, result.stderr + assert upload.__doc__ in result.stdout + + assert result.stdout == cli.invoke(APP, "upload --help").stdout + + +def test_upload_filepath( + cli: CliRunner, static_dir: Path, mock_entities_collection: Collection +) -> None: + """Test upload with a filepath.""" + import json + + from dlite_entities_service.utils_cli import main + + result = cli.invoke( + main.APP, f"upload --file {static_dir / 'valid_entities' / 'Person.json'}" + ) + assert result.exit_code == 0, result.stderr + + assert mock_entities_collection.count_documents({}) == 1 + stored_entity: dict[str, Any] = mock_entities_collection.find_one({}) + stored_entity.pop("_id") + assert stored_entity == json.loads( + (static_dir / "valid_entities" / "Person.json").read_bytes() + ) + + assert "Successfully uploaded 1 entities:" in result.stdout + + +def test_upload_filepath_invalid(cli: CliRunner, static_dir: Path) -> None: + """Test upload with an invalid filepath.""" + from dlite_entities_service.utils_cli.main import APP + + result = cli.invoke( + APP, f"upload --file {static_dir / 'invalid_entities' / 'Person.json'}" + ) + assert result.exit_code == 1, result.stdout + assert "cannot be loaded with DLite." in result.stderr + assert not result.stdout + + +def test_upload_filepath_invalid_format(cli: CliRunner, tmp_path: Path) -> None: + """Test upload with an invalid file format.""" + from dlite_entities_service.utils_cli.main import APP + + (tmp_path / "Person.txt").touch() + + result = cli.invoke(APP, f"upload --file {tmp_path / 'Person.txt'}") + assert result.exit_code == 0, result.stderr + assert "File format 'txt' is not supported." in result.stderr + assert "No entities were uploaded." in result.stdout + + +def test_upload_no_file_or_dir(cli: CliRunner) -> None: + """Test error when no file or directory is provided.""" + from dlite_entities_service.utils_cli.main import APP + + result = cli.invoke(APP, "upload --format json") + assert result.exit_code == 1, result.stdout + assert "Missing either option '--file' / '-f'" in result.stderr + assert not result.stdout + + +def test_upload_directory( + cli: CliRunner, static_dir: Path, mock_entities_collection: Collection +) -> None: + """Test upload with a directory.""" + import json + + from dlite_entities_service.utils_cli import main + + result = cli.invoke(main.APP, f"upload --dir {static_dir / 'valid_entities'}") + assert result.exit_code == 0, result.stderr + + assert mock_entities_collection.count_documents({}) == 3 + stored_entities = list(mock_entities_collection.find({})) + for stored_entity in stored_entities: + stored_entity.pop("_id") + for sample_file in ("Person.json", "Dog.json", "Cat.json"): + assert ( + json.loads((static_dir / "valid_entities" / sample_file).read_bytes()) + in stored_entities + ) + + assert "Successfully uploaded 3 entities:" in result.stdout