From 9f0a4d23b8db93df7f8fdc17a599c5e0830d985d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jos=C3=A9=20Juli=C3=A1n=20Espina?= Date: Tue, 17 Dec 2024 16:46:02 -0600 Subject: [PATCH 1/3] feat: remove peer relation and improve integration tests --- .github/workflows/ci.yaml | 2 +- charmcraft.yaml | 41 ++- justfile | 2 +- .../v0/{interfaces.py => filesystem_info.py} | 212 +++++++-------- src/charm.py | 108 +++----- src/utils/manager.py | 241 +++++++----------- tests/integration/server/charmcraft.yaml | 6 +- .../v0/{interfaces.py => filesystem_info.py} | 212 +++++++-------- tests/integration/server/requirements.txt | 3 +- tests/integration/server/src/charm.py | 160 ++++++++++-- tests/integration/test_charm.py | 62 ++--- tests/unit/test_interfaces.py | 10 +- uv.lock | 12 +- 13 files changed, 528 insertions(+), 543 deletions(-) rename lib/charms/filesystem_client/v0/{interfaces.py => filesystem_info.py} (74%) rename tests/integration/server/lib/charms/filesystem_client/v0/{interfaces.py => filesystem_info.py} (74%) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 6f5942a..af3b333 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -80,7 +80,7 @@ jobs: with: version: 0.5.8 - name: Run tests - run: just type + run: just typecheck integration-test: name: Integration tests (LXD) diff --git a/charmcraft.yaml b/charmcraft.yaml index 0284f5d..c2efd77 100644 --- a/charmcraft.yaml +++ b/charmcraft.yaml @@ -33,38 +33,35 @@ parts: subordinate: true -peers: - storage-peers: - interface: storage_peers - requires: - fs-share: - interface: fs_share + filesystem: + interface: filesystem_info juju-info: interface: juju-info scope: container config: options: - mountinfo: + mounts: default: "{}" type: string description: | - Information to mount the filesystem on the machine. This is specified as a JSON object string. + Information to mount filesystems on the machine. This is specified as a JSON object string. Example usage: ```bash - $ juju config slurmd mountpoints=< None: super().__init__(*args, **kwargs) - # Charm events defined in the FsRequires class. - self._fs = FsRequires(self, "fs-share") + # Charm events defined in the FilesystemRequires class. + self._fs = FilesystemRequires(self, "filesystem") self.framework.observe( - self._fs.on.mount_fs, - self._on_mount_fs, + self._fs.on.mount_filesystem, + self._on_mount_filesystem, ) - def _on_mount_fs(self, event: MountShareEvent) -> None: + def _on_mount_filesystem(self, event: MountFilesystemEvent) -> None: # Handle when new filesystem server is connected. - fs_info = event.fs_info + endpoint = event.endpoint - self.mount("/mnt", fs_info) + self.mount("/mnt", endpoint.info) self.unit.status = ops.ActiveStatus("Mounted filesystem at `/mnt`.") ``` -## FsProvides (filesystem server) +## FilesystemProvides (filesystem server) This library provides a uniform interface for charms that expose filesystems. > __Note:__ It is the responsibility of the provider charm to have -> the implementation for creating a new filesystem share. FsProvides just exposes +> the implementation for creating a new filesystem share. FilesystemProvides just exposes > the interface for the integration. ### Example ```python import ops -from charms.filesystem_client.v0.fs_interfaces import ( - FsProvides, +from charms.filesystem_client.v0.filesystem_info import ( + FilesystemProvides, NfsInfo, ) class StorageServerCharm(ops.CharmBase): - def __init__(self, framework: ops.Framework): + def __init__(self, framework: ops.Framework) -> None: super().__init__(framework) - self._fs = FsProvides(self, "fs-share", "server-peers") + self._filesystem = FilesystemProvides(self, "filesystem", "server-peers") framework.observe(self.on.start, self._on_start) - def _on_start(self, event: ops.StartEvent): + def _on_start(self, event: ops.StartEvent) -> None: # Handle start event. - self._fs.set_fs_info(NfsInfo("192.168.1.254", 65535, "/srv")) + self._filesystem.set_info(NfsInfo("192.168.1.254", 65535, "/srv")) self.unit.status = ops.ActiveStatus() ``` """ @@ -101,7 +100,7 @@ def _on_start(self, event: ops.StartEvent): from abc import ABC, abstractmethod from dataclasses import dataclass from ipaddress import AddressValueError, IPv6Address -from typing import List, Optional, TypeVar +from typing import List, Optional, TypeVar, Self from urllib.parse import parse_qs, quote, unquote, urlencode, urlparse, urlunsplit import ops @@ -117,16 +116,18 @@ def _on_start(self, event: ops.StartEvent): from ops.model import Model, Relation __all__ = [ - "InterfacesError", - "ParseError", - "FsInfo", + "FilesystemInfoError", + "ParseUriError", + "FilesystemInfo", "NfsInfo", "CephfsInfo", "Endpoint", - "MountFsEvent", - "UmountFsEvent", - "FsRequires", - "FsProvides", + "FilesystemEvent", + "MountFilesystemEvent", + "UmountFilesystemEvent", + "FilesystemRequiresEvents", + "FilesystemRequires", + "FilesystemProvides", ] # The unique Charmhub library identifier, never change it @@ -142,11 +143,11 @@ def _on_start(self, event: ops.StartEvent): _logger = logging.getLogger(__name__) -class InterfacesError(Exception): +class FilesystemInfoError(Exception): """Exception raised when an operation failed.""" -class ParseError(InterfacesError): +class ParseUriError(FilesystemInfoError): """Exception raised when a parse operation from an URI failed.""" @@ -213,11 +214,11 @@ def __init__( user: str = "", path: str = "/", options: dict[str, str] = {}, - ): + ) -> None: if not scheme: - raise InterfacesError("scheme cannot be empty") + raise FilesystemInfoError("scheme cannot be empty") if len(hosts) == 0: - raise InterfacesError("list of hosts cannot be empty") + raise FilesystemInfoError("list of hosts cannot be empty") # Strictly convert to the required types to avoid passing through weird data. object.__setattr__(self, "scheme", str(scheme)) @@ -239,7 +240,7 @@ def from_uri(cls, uri: str) -> "_UriData": hostname = unquote(result.hostname or "") if not hostname or hostname[0] != "(" or hostname[-1] != ")": - raise ParseError(f"invalid list of hosts for endpoint `{uri}`") + raise ParseUriError(f"invalid list of hosts for endpoint `{uri}`") hosts = hostname[1:-1].split(",") path = unquote(result.path or "") @@ -249,11 +250,11 @@ def from_uri(cls, uri: str) -> "_UriData": for key, values in parse_qs(result.query, strict_parsing=True).items() } except ValueError: - raise ParseError(f"invalid options for endpoint `{uri}`") + raise ParseUriError(f"invalid options for endpoint `{uri}`") try: return _UriData(scheme=scheme, user=user, hosts=hosts, path=path, options=options) - except InterfacesError as e: - raise ParseError(*e.args) + except FilesystemInfoError as e: + raise ParseUriError(*e.args) def __str__(self) -> str: user = quote(self.user) @@ -268,14 +269,14 @@ def _hostinfo(host: str) -> tuple[str, Optional[int]]: """Parse a host string into the hostname and the port.""" _logger.debug(f"_hostinfo: parsing `{host}`") if len(host) == 0: - raise ParseError("invalid empty host") + raise ParseUriError("invalid empty host") pos = 0 if host[pos] == "[": # IPv6 pos = host.find("]", pos) if pos == -1: - raise ParseError("unclosed bracket for host") + raise ParseUriError("unclosed bracket for host") hostname = host[1:pos] pos = pos + 1 else: @@ -291,19 +292,19 @@ def _hostinfo(host: str) -> tuple[str, Optional[int]]: # more characters after the hostname <==> port if host[pos] != ":": - raise ParseError("expected `:` after IPv6 address") + raise ParseUriError("expected `:` after IPv6 address") try: port = int(host[pos + 1 :]) except ValueError: - raise ParseError("expected int after `:` in host") + raise ParseUriError("expected int after `:` in host") return hostname, port -T = TypeVar("T", bound="FsInfo") +T = TypeVar("T", bound="FilesystemInfo") -class FsInfo(ABC): +class FilesystemInfo(ABC): """Information to mount a filesystem. This is an abstract class that exposes a set of required methods. All filesystems that @@ -313,14 +314,14 @@ class FsInfo(ABC): @classmethod @abstractmethod def from_uri(cls: type[T], uri: str, model: Model) -> T: - """Convert an URI string into a `FsInfo` object.""" + """Convert an URI string into a `FilesystemInfo` object.""" @abstractmethod def to_uri(self, model: Model) -> str: - """Convert this `FsInfo` object into an URI string.""" + """Convert this `FilesystemInfo` object into an URI string.""" - def grant(self, model: Model, relation: ops.Relation): - """Grant permissions for a certain relation to any secrets that this `FsInfo` has. + def grant(self, model: Model, relation: ops.Relation) -> None: + """Grant permissions for a certain relation to any secrets that this `FilesystemInfo` has. This is an optional method because not all filesystems will require secrets to be mounted on the client. @@ -328,12 +329,12 @@ def grant(self, model: Model, relation: ops.Relation): @classmethod @abstractmethod - def fs_type(cls) -> str: + def filesystem_type(cls) -> str: """Get the string identifier of this filesystem type.""" @dataclass(frozen=True) -class NfsInfo(FsInfo): +class NfsInfo(FilesystemInfo): """Information required to mount an NFS share.""" hostname: str @@ -347,14 +348,14 @@ class NfsInfo(FsInfo): @classmethod def from_uri(cls, uri: str, _model: Model) -> "NfsInfo": - """See :py:meth:`FsInfo.from_uri` for documentation on this method.""" + """See :py:meth:`FilesystemInfo.from_uri` for documentation on this method.""" _logger.debug(f"NfsInfo.from_uri: parsing `{uri}`") info = _UriData.from_uri(uri) - if info.scheme != cls.fs_type(): - raise ParseError( - "could not parse `EndpointInfo` with incompatible scheme into `NfsInfo`" + if info.scheme != cls.filesystem_type(): + raise ParseUriError( + "could not parse uri with incompatible scheme into `NfsInfo`" ) path = info.path @@ -372,7 +373,7 @@ def from_uri(cls, uri: str, _model: Model) -> "NfsInfo": return NfsInfo(hostname=hostname, port=port, path=path) def to_uri(self, _model: Model) -> str: - """See :py:meth:`FsInfo.to_uri` for documentation on this method.""" + """See :py:meth:`FilesystemInfo.to_uri` for documentation on this method.""" try: IPv6Address(self.hostname) host = f"[{self.hostname}]" @@ -381,16 +382,16 @@ def to_uri(self, _model: Model) -> str: hosts = [host + f":{self.port}" if self.port else ""] - return str(_UriData(scheme=self.fs_type(), hosts=hosts, path=self.path)) + return str(_UriData(scheme=self.filesystem_type(), hosts=hosts, path=self.path)) @classmethod - def fs_type(cls) -> str: - """See :py:meth:`FsInfo.fs_type` for documentation on this method.""" + def filesystem_type(cls) -> str: + """See :py:meth:`FilesystemInfo.fs_type` for documentation on this method.""" return "nfs" @dataclass(frozen=True) -class CephfsInfo(FsInfo): +class CephfsInfo(FilesystemInfo): """Information required to mount a CephFS share.""" fsid: str @@ -413,35 +414,33 @@ class CephfsInfo(FsInfo): @classmethod def from_uri(cls, uri: str, model: Model) -> "CephfsInfo": - """See :py:meth:`FsInfo.from_uri` for documentation on this method.""" + """See :py:meth:`FilesystemInfo.from_uri` for documentation on this method.""" _logger.debug(f"CephfsInfo.from_uri: parsing `{uri}`") info = _UriData.from_uri(uri) - if info.scheme != cls.fs_type(): - raise ParseError( - "could not parse `EndpointInfo` with incompatible scheme into `CephfsInfo`" - ) + if info.scheme != cls.filesystem_type(): + raise ParseUriError("could not parse uri with incompatible scheme into `CephfsInfo`") path = info.path if not (user := info.user): - raise ParseError("missing user in uri for `CephfsInfo") + raise ParseUriError("missing user in uri for `CephfsInfo") if not (name := info.options.get("name")): - raise ParseError("missing name in uri for `CephfsInfo`") + raise ParseUriError("missing name in uri for `CephfsInfo`") if not (fsid := info.options.get("fsid")): - raise ParseError("missing fsid in uri for `CephfsInfo`") + raise ParseUriError("missing fsid in uri for `CephfsInfo`") monitor_hosts = info.hosts if not (auth := info.options.get("auth")): - raise ParseError("missing auth info in uri for `CephsInfo`") + raise ParseUriError("missing auth info in uri for `CephsInfo`") try: kind, data = auth.split(":", 1) except ValueError: - raise ParseError("Could not get the kind of auth info") + raise ParseUriError("could not get the kind of auth info") if kind == "secret": key = model.get_secret(id=auth).get_content(refresh=True)["key"] @@ -450,14 +449,14 @@ def from_uri(cls, uri: str, model: Model) -> "CephfsInfo": # they don't support secrets. key = data else: - raise ParseError(f"Invalid kind `{kind}` for auth info") + raise ParseUriError(f"invalid kind `{kind}` for auth info") return CephfsInfo( fsid=fsid, name=name, path=path, monitor_hosts=monitor_hosts, user=user, key=key ) def to_uri(self, model: Model) -> str: - """See :py:meth:`FsInfo.to_uri` for documentation on this method.""" + """See :py:meth:`FilesystemInfo.to_uri` for documentation on this method.""" secret = self._get_or_create_auth_secret(model) options = { @@ -469,7 +468,7 @@ def to_uri(self, model: Model) -> str: return str( _UriData( - scheme=self.fs_type(), + scheme=self.filesystem_type(), hosts=self.monitor_hosts, path=self.path, user=self.user, @@ -477,15 +476,15 @@ def to_uri(self, model: Model) -> str: ) ) - def grant(self, model: Model, relation: Relation): - """See :py:meth:`FsInfo.grant` for documentation on this method.""" + def grant(self, model: Model, relation: Relation) -> None: + """See :py:meth:`FilesystemInfo.grant` for documentation on this method.""" secret = self._get_or_create_auth_secret(model) secret.grant(relation) @classmethod - def fs_type(cls) -> str: - """See :py:meth:`FsInfo.fs_type` for documentation on this method.""" + def filesystem_type(cls) -> str: + """See :py:meth:`FilesystemInfo.fs_type` for documentation on this method.""" return "cephfs" def _get_or_create_auth_secret(self, model: Model) -> ops.Secret: @@ -505,25 +504,28 @@ def _get_or_create_auth_secret(self, model: Model) -> ops.Secret: class Endpoint: """Endpoint data exposed by a filesystem server.""" - fs_info: FsInfo + info: FilesystemInfo """Filesystem information required to mount this endpoint.""" uri: str """Raw URI exposed by this endpoint.""" + # Right now this is unused on the client, but having the raw uri + # available was useful on a previous version of the charm, so leaving + # this exposed just in case we need it in the future. -def _uri_to_fs_info(uri: str, model: Model) -> FsInfo: +def _uri_to_fs_info(uri: str, model: Model) -> FilesystemInfo: scheme = uri.split("://", maxsplit=1)[0] - if scheme == NfsInfo.fs_type(): + if scheme == NfsInfo.filesystem_type(): return NfsInfo.from_uri(uri, model) - elif scheme == CephfsInfo.fs_type(): + elif scheme == CephfsInfo.filesystem_type(): return CephfsInfo.from_uri(uri, model) else: - raise InterfacesError(f"unsupported filesystem type `{scheme}`") + raise FilesystemInfoError(f"unsupported filesystem type `{scheme}`") -class _MountEvent(RelationEvent): - """Base event for mount-related events.""" +class FilesystemEvent(RelationEvent): + """Base event for filesystem-related events.""" @property def endpoint(self) -> Optional[Endpoint]: @@ -533,19 +535,19 @@ def endpoint(self) -> Optional[Endpoint]: return Endpoint(_uri_to_fs_info(uri, self.framework.model), uri) -class MountFsEvent(_MountEvent): +class MountFilesystemEvent(FilesystemEvent): """Emit when a filesystem is ready to be mounted.""" -class UmountFsEvent(_MountEvent): +class UmountFilesystemEvent(FilesystemEvent): """Emit when a filesystem needs to be unmounted.""" -class _FsRequiresEvents(CharmEvents): +class FilesystemRequiresEvents(CharmEvents): """Events that FS servers can emit.""" - mount_fs = EventSource(MountFsEvent) - umount_fs = EventSource(UmountFsEvent) + mount_filesystem = EventSource(MountFilesystemEvent) + umount_filesystem = EventSource(UmountFilesystemEvent) class _BaseInterface(Object): @@ -572,10 +574,10 @@ def relations(self) -> List[Relation]: return result -class FsRequires(_BaseInterface): +class FilesystemRequires(_BaseInterface): """Consumer-side interface of filesystem integrations.""" - on = _FsRequiresEvents() + on = FilesystemRequiresEvents() def __init__(self, charm: CharmBase, relation_name: str) -> None: super().__init__(charm, relation_name) @@ -586,13 +588,13 @@ def __init__(self, charm: CharmBase, relation_name: str) -> None: def _on_relation_changed(self, event: RelationChangedEvent) -> None: """Handle when the databag between client and server has been updated.""" - _logger.debug("Emitting `MountShare` event from `RelationChanged` hook") - self.on.mount_fs.emit(event.relation, app=event.app, unit=event.unit) + _logger.debug("emitting `MountFilesystem` event from `RelationChanged` hook") + self.on.mount_filesystem.emit(event.relation, app=event.app, unit=event.unit) def _on_relation_departed(self, event: RelationDepartedEvent) -> None: """Handle when server departs integration.""" - _logger.debug("Emitting `UmountShare` event from `RelationDeparted` hook") - self.on.umount_fs.emit(event.relation, app=event.app, unit=event.unit) + _logger.debug("emitting `UmountFilesystem` event from `RelationDeparted` hook") + self.on.umount_filesystem.emit(event.relation, app=event.app, unit=event.unit) @property def endpoints(self) -> List[Endpoint]: @@ -601,11 +603,11 @@ def endpoints(self) -> List[Endpoint]: for relation in self.relations: if not (uri := relation.data[relation.app].get("endpoint")): continue - result.append(Endpoint(fs_info=_uri_to_fs_info(uri, self.model), uri=uri)) + result.append(Endpoint(info=_uri_to_fs_info(uri, self.model), uri=uri)) return result -class FsProvides(_BaseInterface): +class FilesystemProvides(_BaseInterface): """Provider-side interface of filesystem integrations.""" def __init__(self, charm: CharmBase, relation_name: str, peer_relation_name: str) -> None: @@ -613,11 +615,11 @@ def __init__(self, charm: CharmBase, relation_name: str, peer_relation_name: str self._peer_relation_name = peer_relation_name self.framework.observe(charm.on[relation_name].relation_joined, self._update_relation) - def set_fs_info(self, fs_info: FsInfo) -> None: + def set_info(self, info: FilesystemInfo) -> None: """Set information to mount a filesystem. Args: - fs_info: Information required to mount the filesystem. + info: Information required to mount the filesystem. Notes: Only the application leader unit can set the filesystem data. @@ -625,12 +627,12 @@ def set_fs_info(self, fs_info: FsInfo) -> None: if not self.unit.is_leader(): return - uri = fs_info.to_uri(self.model) + uri = info.to_uri(self.model) self._endpoint = uri for relation in self.relations: - fs_info.grant(self.model, relation) + info.grant(self.model, relation) relation.data[self.app]["endpoint"] = uri def _update_relation(self, event: RelationJoinedEvent) -> None: @@ -666,8 +668,8 @@ def _get_state(self, key: str) -> Optional[str]: def _set_state(self, key: str, data: str) -> None: """Insert a value into the global state.""" if not self._peers: - raise InterfacesError( - "Peer relation can only be accessed after the relation is established" + raise FilesystemInfoError( + "peer relation can only be accessed after the relation is established" ) self._peers.data[self.app][key] = data diff --git a/src/charm.py b/src/charm.py index 9bbe0de..93c0e5e 100755 --- a/src/charm.py +++ b/src/charm.py @@ -7,12 +7,9 @@ import json import logging from collections import Counter -from contextlib import contextmanager -from typing import Any, Generator, Optional -import charms.operator_libs_linux.v0.apt as apt import ops -from charms.filesystem_client.v0.interfaces import FsRequires +from charms.filesystem_client.v0.filesystem_info import FilesystemRequires from jsonschema import ValidationError, validate from utils.manager import MountsManager @@ -35,12 +32,13 @@ }, } -PEER_NAME = "storage-peers" - class StopCharmError(Exception): """Exception raised when a method needs to finish the execution of the charm code.""" + def __init__(self, status: ops.StatusBase): + self.status = status + # Trying to use a delta charm (one method per event) proved to be a bit unwieldy, since # we would have to handle multiple updates at once: @@ -60,13 +58,13 @@ class FilesystemClientCharm(ops.CharmBase): def __init__(self, framework: ops.Framework): super().__init__(framework) - self._fs_share = FsRequires(self, "fs-share") - self._mounts_manager = MountsManager() + self._filesystems = FilesystemRequires(self, "filesystem") + self._mounts_manager = MountsManager(self) framework.observe(self.on.upgrade_charm, self._handle_event) framework.observe(self.on.update_status, self._handle_event) framework.observe(self.on.config_changed, self._handle_event) - framework.observe(self._fs_share.on.mount_fs, self._handle_event) - framework.observe(self._fs_share.on.umount_fs, self._handle_event) + framework.observe(self._filesystems.on.mount_filesystem, self._handle_event) + framework.observe(self._filesystems.on.umount_filesystem, self._handle_event) def _handle_event(self, event: ops.EventBase) -> None: # noqa: C901 try: @@ -75,23 +73,24 @@ def _handle_event(self, event: ops.EventBase) -> None: # noqa: C901 self._ensure_installed() config = self._get_config() self._mount_filesystems(config) - except StopCharmError: + except StopCharmError as e: # This was the cleanest way to ensure the inner methods can still return prematurely # when an error occurs. + self.app.status = e.status return - self.unit.status = ops.ActiveStatus("Mounted shares.") + self.unit.status = ops.ActiveStatus("Mounted filesystems.") def _ensure_installed(self): """Ensure the required packages are installed into the unit.""" if not self._mounts_manager.installed: self.unit.status = ops.MaintenanceStatus("Installing required packages.") - self._mounts_manager.ensure(apt.PackageState.Present) + self._mounts_manager.install() def _get_config(self) -> dict[str, dict[str, str | bool]]: """Get and validate the configuration of the charm.""" try: - config = json.loads(str(self.config.get("mountinfo", ""))) + config = json.loads(str(self.config.get("mounts", ""))) validate(config, CONFIG_SCHEMA) config: dict[str, dict[str, str | bool]] = config for fs, opts in config.items(): @@ -99,39 +98,30 @@ def _get_config(self) -> dict[str, dict[str, str | bool]]: opts[opt] = opts.get(opt, False) return config except (json.JSONDecodeError, ValidationError) as e: - self.app.status = ops.BlockedStatus( - f"invalid configuration for option `mountinfo`. reason: {e}" + raise StopCharmError( + ops.BlockedStatus(f"invalid configuration for option `mounts`. reason:\n{e}") ) - raise StopCharmError() def _mount_filesystems(self, config: dict[str, dict[str, str | bool]]): """Mount all available filesystems for the charm.""" - shares = self._fs_share.endpoints - active_filesystems = set() - for fs_type, count in Counter([share.fs_info.fs_type() for share in shares]).items(): + endpoints = self._filesystems.endpoints + for fs_type, count in Counter( + [endpoint.info.filesystem_type() for endpoint in endpoints] + ).items(): if count > 1: - self.app.status = ops.BlockedStatus( - f"Too many relations for mount type `{fs_type}`." + raise StopCharmError( + ops.BlockedStatus(f"Too many relations for mount type `{fs_type}`.") ) - raise StopCharmError() - active_filesystems.add(fs_type) - - with self.mounts() as mounts: - # Cleanup and unmount all the mounts that are not available. - for fs_type in list(mounts.keys()): - if fs_type not in active_filesystems: - self._mounts_manager.umount(str(mounts[fs_type]["mountpoint"])) - del mounts[fs_type] - - for share in shares: - fs_type = share.fs_info.fs_type() + + self.unit.status = ops.MaintenanceStatus("Ensuring filesystems are mounted.") + + with self._mounts_manager.mounts() as mounts: + for endpoint in endpoints: + fs_type = endpoint.info.filesystem_type() if not (options := config.get(fs_type)): - self.app.status = ops.BlockedStatus( - f"Missing configuration for mount type `{fs_type}." + raise StopCharmError( + ops.BlockedStatus(f"Missing configuration for mount type `{fs_type}`.") ) - raise StopCharmError() - - options["uri"] = share.uri mountpoint = str(options["mountpoint"]) @@ -140,45 +130,7 @@ def _mount_filesystems(self, config: dict[str, dict[str, str | bool]]): opts.append("nosuid" if options.get("nosuid") else "suid") opts.append("nodev" if options.get("nodev") else "dev") opts.append("ro" if options.get("read-only") else "rw") - - self.unit.status = ops.MaintenanceStatus(f"Mounting `{mountpoint}`") - - if not (mount := mounts.get(fs_type)) or mount != options: - # Just in case, unmount the previously mounted share - if mount: - self._mounts_manager.umount(str(mount["mountpoint"])) - self._mounts_manager.mount(share.fs_info, mountpoint, options=opts) - mounts[fs_type] = options - - @property - def peers(self) -> Optional[ops.Relation]: - """Fetch the peer relation.""" - return self.model.get_relation(PEER_NAME) - - @contextmanager - def mounts(self) -> Generator[dict[str, dict[str, str | bool]], None, None]: - """Get the mounted filesystems.""" - mounts = self.get_state("mounts") - yield mounts - # Don't set the state if the program throws an error. - # This guarantees we're in a clean state after the charm unblocks. - self.set_state("mounts", mounts) - - def set_state(self, key: str, data: Any) -> None: - """Insert a value into the global state.""" - if not self.peers: - raise RuntimeError( - "Peer relation can only be written to after the relation is established" - ) - self.peers.data[self.app][key] = json.dumps(data) - - def get_state(self, key: str) -> dict[Any, Any]: - """Get a value from the global state.""" - if not self.peers: - return {} - - data = self.peers.data[self.app].get(key, "{}") - return json.loads(data) + mounts.add(info=endpoint.info, mountpoint=mountpoint, options=opts) if __name__ == "__main__": # pragma: nocover diff --git a/src/utils/manager.py b/src/utils/manager.py index 423df74..3a9fe7a 100644 --- a/src/utils/manager.py +++ b/src/utils/manager.py @@ -3,18 +3,19 @@ """Manage machine mounts and dependencies.""" +import contextlib import logging import os import pathlib -import shutil import subprocess from dataclasses import dataclass from ipaddress import AddressValueError, IPv6Address -from typing import Iterator, List, Optional, Union +from typing import Generator, List, Optional, Union import charms.operator_libs_linux.v0.apt as apt import charms.operator_libs_linux.v1.systemd as systemd -from charms.filesystem_client.v0.interfaces import CephfsInfo, FsInfo, NfsInfo +import ops +from charms.filesystem_client.v0.filesystem_info import CephfsInfo, FilesystemInfo, NfsInfo _logger = logging.getLogger(__name__) @@ -53,15 +54,57 @@ class MountInfo: passno: str +@dataclass +class _MountInfo: + endpoint: str + options: [str] + + +class Mounts: + """Collection of mounts that need to be managed by the `MountsManager`.""" + + _mounts: dict[str, _MountInfo] + + def __init__(self): + self._mounts = {} + + def add( + self, + info: FilesystemInfo, + mountpoint: Union[str, os.PathLike], + options: Optional[List[str]] = None, + ) -> None: + """Add a mount to the list of managed mounts. + + Args: + info: Share information required to mount the share. + mountpoint: System location to mount the share. + options: Mount options to pass when mounting the share. + + Raises: + Error: Raised if the mount operation fails. + """ + if options is None: + options = [] + + endpoint, additional_opts = _get_endpoint_and_opts(info) + options = sorted(options + additional_opts) + + self._mounts[str(mountpoint)] = _MountInfo(endpoint=endpoint, options=options) + + class MountsManager: """Manager for mounted filesystems in the current system.""" - def __init__(self): + def __init__(self, charm: ops.CharmBase): # Lazily initialized self._pkgs = None + self._master_file = pathlib.Path(f"/etc/auto.master.d/{charm.app.name}.autofs") + self._autofs_file = pathlib.Path(f"/etc/auto.{charm.app.name}") @property def _packages(self) -> List[apt.DebianPackage]: + """List of packages required by the client.""" if not self._pkgs: self._pkgs = [ apt.DebianPackage.from_system(pkg) @@ -75,23 +118,35 @@ def installed(self) -> bool: for pkg in self._packages: if not pkg.present: return False + + if not self._master_file.exists or not self._autofs_file.exists: + return False + return True - def ensure(self, state: apt.PackageState) -> None: - """Ensure that the mount packages are in the specified state. + def install(self): + """Install the required mount packages. Raises: Error: Raised if this failed to change the state of any of the required packages. """ try: for pkg in self._packages: - pkg.ensure(state) + pkg.ensure(apt.PackageState.Present) except (apt.PackageError, apt.PackageNotFoundError) as e: _logger.error( - f"failed to change the state of the required packages. Reason:\n{e.message}" + f"failed to change the state of the required packages. reason:\n{e.message}" ) raise Error(e.message) + try: + self._master_file.touch(mode=0o600) + self._autofs_file.touch(mode=0o600) + self._master_file.write_text(f"/- {self._autofs_file}") + except IOError as e: + _logger.error(f"failed to create the required autofs files. reason:\n{e}") + raise Error("failed to create the required autofs files") + def supported(self) -> bool: """Check if underlying base supports mounting shares.""" try: @@ -107,156 +162,46 @@ def supported(self) -> bool: _logger.warning("Could not detect execution in virtualized environment") return True - def fetch(self, target: str) -> Optional[MountInfo]: - """Fetch information about a mount. + @contextlib.contextmanager + def mounts(self, force_mount=False) -> Generator[Mounts, None, None]: + """Get the list of `Mounts` that need to be managed by the `MountsManager`. - Args: - target: share mountpoint information to fetch. - - Returns: - Optional[MountInfo]: Mount information. None if share is not mounted. + It will initially contain no mounts, and any mount that is added to + `Mounts` will be mounted by the manager. Mounts that were + added on previous executions will get removed if they're not added again + to the `Mounts` object. """ - # We need to trigger an automount for the mounts that are of type `autofs`, - # since those could contain an unlisted mount. - _trigger_autofs() - - for mount in _mounts(): - if mount.mountpoint == target: - return mount - - return None - - def mounts(self) -> List[MountInfo]: - """Get all mounts on a machine. - - Returns: - List[MountInfo]: All current mounts on machine. - """ - _trigger_autofs() - - return list(_mounts("autofs")) - - def mounted(self, target: str) -> bool: - """Determine if mountpoint is mounted. - - Args: - target: share mountpoint to check. - """ - return self.fetch(target) is not None - - def mount( - self, - share_info: FsInfo, - mountpoint: Union[str, os.PathLike], - options: Optional[List[str]] = None, - ) -> None: - """Mount a share. - - Args: - share_info: Share information required to mount the share. - mountpoint: System location to mount the share. - options: Mount options to pass when mounting the share. - - Raises: - Error: Raised if the mount operation fails. - """ - if options is None: - options = [] - # Try to create the mountpoint without checking if it exists to avoid TOCTOU. - target = pathlib.Path(mountpoint) - try: - target.mkdir() - _logger.debug(f"Created mountpoint {mountpoint}.") - except FileExistsError: - _logger.warning(f"Mountpoint {mountpoint} already exists.") - - endpoint, additional_opts = _get_endpoint_and_opts(share_info) - options = options + additional_opts - - _logger.debug(f"Mounting share {endpoint} at {target}") - autofs_id = _mountpoint_to_autofs_id(target) - pathlib.Path(f"/etc/auto.master.d/{autofs_id}.autofs").write_text( - f"/- /etc/auto.{autofs_id}" - ) - pathlib.Path(f"/etc/auto.{autofs_id}").write_text( - f"{target} -{','.join(options)} {endpoint}" + mounts = Mounts() + yield mounts + # This will not resume if the caller raised an exception, which + # should be enough to ensure the file is not written if the charm entered + # an error state. + new_autofs = "\n".join( + ( + f"{mountpoint} -{','.join(info.options)} {info.endpoint}" + for mountpoint, info in sorted(mounts._mounts.items()) + ) ) - try: - systemd.service_reload("autofs", restart_on_failure=True) - except systemd.SystemdError as e: - _logger.error(f"Failed to mount {endpoint} at {target}. Reason:\n{e}") - if "Operation not permitted" in str(e) and not self.supported(): - raise Error("Mounting shares not supported on LXD containers") - raise Error(f"Failed to mount {endpoint} at {target}") - - def umount(self, mountpoint: Union[str, os.PathLike]) -> None: - """Unmount a share. - - Args: - mountpoint: share mountpoint to unmount. + old_autofs = self._autofs_file.read_text() - Raises: - Error: Raised if the unmount operation fails. - """ - _logger.debug(f"Unmounting share at mountpoint {mountpoint}") - autofs_id = _mountpoint_to_autofs_id(mountpoint) - pathlib.Path(f"/etc/auto.{autofs_id}").unlink(missing_ok=True) - pathlib.Path(f"/etc/auto.master.d/{autofs_id}.autofs").unlink(missing_ok=True) + # Avoid restarting autofs if the config didn't change. + if not force_mount and new_autofs == old_autofs: + return try: + for mount in mounts._mounts.keys(): + pathlib.Path(mount).mkdir(parents=True, exist_ok=True) + self._autofs_file.write_text(new_autofs) systemd.service_reload("autofs", restart_on_failure=True) except systemd.SystemdError as e: - _logger.error(f"Failed to unmount {mountpoint}. Reason:\n{e}") - raise Error(f"Failed to unmount {mountpoint}") - - shutil.rmtree(mountpoint, ignore_errors=True) - - -def _trigger_autofs() -> None: - """Triggers a mount on all filesystems handled by autofs. - - This function is useful to make autofs-managed mounts appear on the - `/proc/mount` file, since they could be unmounted when reading the file. - """ - for fs in _mounts("autofs"): - _logger.info(f"triggering automount for `{fs.mountpoint}`") - try: - os.scandir(fs.mountpoint).close() - except OSError as e: - # Not critical since it could also be caused by unrelated mounts, - # but should be good to log it in case this causes problems. - _logger.warning(f"Could not trigger automount for `{fs.mountpoint}`. Reason:\n{e}") - - -def _mountpoint_to_autofs_id(mountpoint: Union[str, os.PathLike]) -> str: - """Get the autofs id of a mountpoint path. - - Args: - mountpoint: share mountpoint. - """ - path = pathlib.Path(mountpoint).resolve() - return str(path).lstrip("/").replace("/", "-") - - -def _mounts(fstype: str = "") -> Iterator[MountInfo]: - """Get an iterator of all mounts in the system that have the requested fstype. - - Returns: - Iterator[MountInfo]: All the mounts with a valid fstype. - """ - with pathlib.Path("/proc/mounts").open("rt") as mounts: - for mount in mounts: - # Lines in /proc/mounts follow the standard format - # - m = MountInfo(*mount.split()) - if fstype and not m.fstype.startswith(fstype): - continue - - yield m + _logger.error(f"failed to mount filesystems. reason:\n{e}") + if "Operation not permitted" in str(e) and not self.supported(): + raise Error("mounting shares not supported on LXD containers") + raise Error("failed to mount filesystems") -def _get_endpoint_and_opts(info: FsInfo) -> tuple[str, [str]]: +def _get_endpoint_and_opts(info: FilesystemInfo) -> tuple[str, [str]]: match info: case NfsInfo(hostname=hostname, port=port, path=path): try: @@ -279,6 +224,6 @@ def _get_endpoint_and_opts(info: FsInfo) -> tuple[str, [str]]: f"secret={secret}", ] case _: - raise Error(f"unsupported filesystem type `{info.fs_type()}`") + raise Error(f"unsupported filesystem type `{info.filesystem_type()}`") return endpoint, options diff --git a/tests/integration/server/charmcraft.yaml b/tests/integration/server/charmcraft.yaml index 92cbf7b..0ad49e7 100644 --- a/tests/integration/server/charmcraft.yaml +++ b/tests/integration/server/charmcraft.yaml @@ -18,11 +18,11 @@ parts: peers: server-peers: - interface: server_peers + interface: server_peers provides: - fs-share: - interface: fs_share + filesystem: + interface: filesystem_info limit: 1 config: diff --git a/tests/integration/server/lib/charms/filesystem_client/v0/interfaces.py b/tests/integration/server/lib/charms/filesystem_client/v0/filesystem_info.py similarity index 74% rename from tests/integration/server/lib/charms/filesystem_client/v0/interfaces.py rename to tests/integration/server/lib/charms/filesystem_client/v0/filesystem_info.py index 4927980..d0e1ea7 100644 --- a/tests/integration/server/lib/charms/filesystem_client/v0/interfaces.py +++ b/tests/integration/server/lib/charms/filesystem_client/v0/filesystem_info.py @@ -14,85 +14,84 @@ """Library to manage integrations between filesystem providers and consumers. -This library contains the FsProvides and FsRequires classes for managing an +This library contains the FilesystemProvides and FilesystemRequires classes for managing an integration between a filesystem server operator and a filesystem client operator. -## FsInfo (filesystem mount information) +## FilesystemInfo (filesystem mount information) This abstract class defines the methods that a filesystem type must expose for providers and consumers. Any subclass of this class will be compatible with the other methods exposed by the interface library, but the server and the client are the ones responsible for deciding which filesystems to support. -## FsRequires (filesystem client) +## FilesystemRequires (filesystem client) This class provides a uniform interface for charms that need to mount or unmount filesystems, and convenience methods for consuming data sent by a filesystem server charm. ### Defined events -- `mount_fs`: Event emitted when the filesystem is ready to be mounted. -- `umount_fs`: Event emitted when the filesystem needs to be unmounted. +- `mount_filesystem`: Event emitted when the filesystem is ready to be mounted. +- `umount_filesystem`: Event emitted when the filesystem needs to be unmounted. ### Example ``python import ops -from charms.storage_libs.v0.fs_interfaces import ( - FsRequires, - MountFsEvent, +from charms.filesystem_client.v0.filesystem_info import ( + FilesystemRequires, + MountFilesystemEvent, ) - class StorageClientCharm(ops.CharmBase): # Application charm that needs to mount filesystems. def __init__(self, *args, **kwargs) -> None: super().__init__(*args, **kwargs) - # Charm events defined in the FsRequires class. - self._fs = FsRequires(self, "fs-share") + # Charm events defined in the FilesystemRequires class. + self._fs = FilesystemRequires(self, "filesystem") self.framework.observe( - self._fs.on.mount_fs, - self._on_mount_fs, + self._fs.on.mount_filesystem, + self._on_mount_filesystem, ) - def _on_mount_fs(self, event: MountShareEvent) -> None: + def _on_mount_filesystem(self, event: MountFilesystemEvent) -> None: # Handle when new filesystem server is connected. - fs_info = event.fs_info + endpoint = event.endpoint - self.mount("/mnt", fs_info) + self.mount("/mnt", endpoint.info) self.unit.status = ops.ActiveStatus("Mounted filesystem at `/mnt`.") ``` -## FsProvides (filesystem server) +## FilesystemProvides (filesystem server) This library provides a uniform interface for charms that expose filesystems. > __Note:__ It is the responsibility of the provider charm to have -> the implementation for creating a new filesystem share. FsProvides just exposes +> the implementation for creating a new filesystem share. FilesystemProvides just exposes > the interface for the integration. ### Example ```python import ops -from charms.filesystem_client.v0.fs_interfaces import ( - FsProvides, +from charms.filesystem_client.v0.filesystem_info import ( + FilesystemProvides, NfsInfo, ) class StorageServerCharm(ops.CharmBase): - def __init__(self, framework: ops.Framework): + def __init__(self, framework: ops.Framework) -> None: super().__init__(framework) - self._fs = FsProvides(self, "fs-share", "server-peers") + self._filesystem = FilesystemProvides(self, "filesystem", "server-peers") framework.observe(self.on.start, self._on_start) - def _on_start(self, event: ops.StartEvent): + def _on_start(self, event: ops.StartEvent) -> None: # Handle start event. - self._fs.set_fs_info(NfsInfo("192.168.1.254", 65535, "/srv")) + self._filesystem.set_info(NfsInfo("192.168.1.254", 65535, "/srv")) self.unit.status = ops.ActiveStatus() ``` """ @@ -101,7 +100,7 @@ def _on_start(self, event: ops.StartEvent): from abc import ABC, abstractmethod from dataclasses import dataclass from ipaddress import AddressValueError, IPv6Address -from typing import List, Optional, TypeVar +from typing import List, Optional, TypeVar, Self from urllib.parse import parse_qs, quote, unquote, urlencode, urlparse, urlunsplit import ops @@ -117,16 +116,18 @@ def _on_start(self, event: ops.StartEvent): from ops.model import Model, Relation __all__ = [ - "InterfacesError", - "ParseError", - "FsInfo", + "FilesystemInfoError", + "ParseUriError", + "FilesystemInfo", "NfsInfo", "CephfsInfo", "Endpoint", - "MountFsEvent", - "UmountFsEvent", - "FsRequires", - "FsProvides", + "FilesystemEvent", + "MountFilesystemEvent", + "UmountFilesystemEvent", + "FilesystemRequiresEvents", + "FilesystemRequires", + "FilesystemProvides", ] # The unique Charmhub library identifier, never change it @@ -142,11 +143,11 @@ def _on_start(self, event: ops.StartEvent): _logger = logging.getLogger(__name__) -class InterfacesError(Exception): +class FilesystemInfoError(Exception): """Exception raised when an operation failed.""" -class ParseError(InterfacesError): +class ParseUriError(FilesystemInfoError): """Exception raised when a parse operation from an URI failed.""" @@ -213,11 +214,11 @@ def __init__( user: str = "", path: str = "/", options: dict[str, str] = {}, - ): + ) -> None: if not scheme: - raise InterfacesError("scheme cannot be empty") + raise FilesystemInfoError("scheme cannot be empty") if len(hosts) == 0: - raise InterfacesError("list of hosts cannot be empty") + raise FilesystemInfoError("list of hosts cannot be empty") # Strictly convert to the required types to avoid passing through weird data. object.__setattr__(self, "scheme", str(scheme)) @@ -239,7 +240,7 @@ def from_uri(cls, uri: str) -> "_UriData": hostname = unquote(result.hostname or "") if not hostname or hostname[0] != "(" or hostname[-1] != ")": - raise ParseError(f"invalid list of hosts for endpoint `{uri}`") + raise ParseUriError(f"invalid list of hosts for endpoint `{uri}`") hosts = hostname[1:-1].split(",") path = unquote(result.path or "") @@ -249,11 +250,11 @@ def from_uri(cls, uri: str) -> "_UriData": for key, values in parse_qs(result.query, strict_parsing=True).items() } except ValueError: - raise ParseError(f"invalid options for endpoint `{uri}`") + raise ParseUriError(f"invalid options for endpoint `{uri}`") try: return _UriData(scheme=scheme, user=user, hosts=hosts, path=path, options=options) - except InterfacesError as e: - raise ParseError(*e.args) + except FilesystemInfoError as e: + raise ParseUriError(*e.args) def __str__(self) -> str: user = quote(self.user) @@ -268,14 +269,14 @@ def _hostinfo(host: str) -> tuple[str, Optional[int]]: """Parse a host string into the hostname and the port.""" _logger.debug(f"_hostinfo: parsing `{host}`") if len(host) == 0: - raise ParseError("invalid empty host") + raise ParseUriError("invalid empty host") pos = 0 if host[pos] == "[": # IPv6 pos = host.find("]", pos) if pos == -1: - raise ParseError("unclosed bracket for host") + raise ParseUriError("unclosed bracket for host") hostname = host[1:pos] pos = pos + 1 else: @@ -291,19 +292,19 @@ def _hostinfo(host: str) -> tuple[str, Optional[int]]: # more characters after the hostname <==> port if host[pos] != ":": - raise ParseError("expected `:` after IPv6 address") + raise ParseUriError("expected `:` after IPv6 address") try: port = int(host[pos + 1 :]) except ValueError: - raise ParseError("expected int after `:` in host") + raise ParseUriError("expected int after `:` in host") return hostname, port -T = TypeVar("T", bound="FsInfo") +T = TypeVar("T", bound="FilesystemInfo") -class FsInfo(ABC): +class FilesystemInfo(ABC): """Information to mount a filesystem. This is an abstract class that exposes a set of required methods. All filesystems that @@ -313,14 +314,14 @@ class FsInfo(ABC): @classmethod @abstractmethod def from_uri(cls: type[T], uri: str, model: Model) -> T: - """Convert an URI string into a `FsInfo` object.""" + """Convert an URI string into a `FilesystemInfo` object.""" @abstractmethod def to_uri(self, model: Model) -> str: - """Convert this `FsInfo` object into an URI string.""" + """Convert this `FilesystemInfo` object into an URI string.""" - def grant(self, model: Model, relation: ops.Relation): - """Grant permissions for a certain relation to any secrets that this `FsInfo` has. + def grant(self, model: Model, relation: ops.Relation) -> None: + """Grant permissions for a certain relation to any secrets that this `FilesystemInfo` has. This is an optional method because not all filesystems will require secrets to be mounted on the client. @@ -328,12 +329,12 @@ def grant(self, model: Model, relation: ops.Relation): @classmethod @abstractmethod - def fs_type(cls) -> str: + def filesystem_type(cls) -> str: """Get the string identifier of this filesystem type.""" @dataclass(frozen=True) -class NfsInfo(FsInfo): +class NfsInfo(FilesystemInfo): """Information required to mount an NFS share.""" hostname: str @@ -347,14 +348,14 @@ class NfsInfo(FsInfo): @classmethod def from_uri(cls, uri: str, _model: Model) -> "NfsInfo": - """See :py:meth:`FsInfo.from_uri` for documentation on this method.""" + """See :py:meth:`FilesystemInfo.from_uri` for documentation on this method.""" _logger.debug(f"NfsInfo.from_uri: parsing `{uri}`") info = _UriData.from_uri(uri) - if info.scheme != cls.fs_type(): - raise ParseError( - "could not parse `EndpointInfo` with incompatible scheme into `NfsInfo`" + if info.scheme != cls.filesystem_type(): + raise ParseUriError( + "could not parse uri with incompatible scheme into `NfsInfo`" ) path = info.path @@ -372,7 +373,7 @@ def from_uri(cls, uri: str, _model: Model) -> "NfsInfo": return NfsInfo(hostname=hostname, port=port, path=path) def to_uri(self, _model: Model) -> str: - """See :py:meth:`FsInfo.to_uri` for documentation on this method.""" + """See :py:meth:`FilesystemInfo.to_uri` for documentation on this method.""" try: IPv6Address(self.hostname) host = f"[{self.hostname}]" @@ -381,16 +382,16 @@ def to_uri(self, _model: Model) -> str: hosts = [host + f":{self.port}" if self.port else ""] - return str(_UriData(scheme=self.fs_type(), hosts=hosts, path=self.path)) + return str(_UriData(scheme=self.filesystem_type(), hosts=hosts, path=self.path)) @classmethod - def fs_type(cls) -> str: - """See :py:meth:`FsInfo.fs_type` for documentation on this method.""" + def filesystem_type(cls) -> str: + """See :py:meth:`FilesystemInfo.fs_type` for documentation on this method.""" return "nfs" @dataclass(frozen=True) -class CephfsInfo(FsInfo): +class CephfsInfo(FilesystemInfo): """Information required to mount a CephFS share.""" fsid: str @@ -413,35 +414,33 @@ class CephfsInfo(FsInfo): @classmethod def from_uri(cls, uri: str, model: Model) -> "CephfsInfo": - """See :py:meth:`FsInfo.from_uri` for documentation on this method.""" + """See :py:meth:`FilesystemInfo.from_uri` for documentation on this method.""" _logger.debug(f"CephfsInfo.from_uri: parsing `{uri}`") info = _UriData.from_uri(uri) - if info.scheme != cls.fs_type(): - raise ParseError( - "could not parse `EndpointInfo` with incompatible scheme into `CephfsInfo`" - ) + if info.scheme != cls.filesystem_type(): + raise ParseUriError("could not parse uri with incompatible scheme into `CephfsInfo`") path = info.path if not (user := info.user): - raise ParseError("missing user in uri for `CephfsInfo") + raise ParseUriError("missing user in uri for `CephfsInfo") if not (name := info.options.get("name")): - raise ParseError("missing name in uri for `CephfsInfo`") + raise ParseUriError("missing name in uri for `CephfsInfo`") if not (fsid := info.options.get("fsid")): - raise ParseError("missing fsid in uri for `CephfsInfo`") + raise ParseUriError("missing fsid in uri for `CephfsInfo`") monitor_hosts = info.hosts if not (auth := info.options.get("auth")): - raise ParseError("missing auth info in uri for `CephsInfo`") + raise ParseUriError("missing auth info in uri for `CephsInfo`") try: kind, data = auth.split(":", 1) except ValueError: - raise ParseError("Could not get the kind of auth info") + raise ParseUriError("could not get the kind of auth info") if kind == "secret": key = model.get_secret(id=auth).get_content(refresh=True)["key"] @@ -450,14 +449,14 @@ def from_uri(cls, uri: str, model: Model) -> "CephfsInfo": # they don't support secrets. key = data else: - raise ParseError(f"Invalid kind `{kind}` for auth info") + raise ParseUriError(f"invalid kind `{kind}` for auth info") return CephfsInfo( fsid=fsid, name=name, path=path, monitor_hosts=monitor_hosts, user=user, key=key ) def to_uri(self, model: Model) -> str: - """See :py:meth:`FsInfo.to_uri` for documentation on this method.""" + """See :py:meth:`FilesystemInfo.to_uri` for documentation on this method.""" secret = self._get_or_create_auth_secret(model) options = { @@ -469,7 +468,7 @@ def to_uri(self, model: Model) -> str: return str( _UriData( - scheme=self.fs_type(), + scheme=self.filesystem_type(), hosts=self.monitor_hosts, path=self.path, user=self.user, @@ -477,15 +476,15 @@ def to_uri(self, model: Model) -> str: ) ) - def grant(self, model: Model, relation: Relation): - """See :py:meth:`FsInfo.grant` for documentation on this method.""" + def grant(self, model: Model, relation: Relation) -> None: + """See :py:meth:`FilesystemInfo.grant` for documentation on this method.""" secret = self._get_or_create_auth_secret(model) secret.grant(relation) @classmethod - def fs_type(cls) -> str: - """See :py:meth:`FsInfo.fs_type` for documentation on this method.""" + def filesystem_type(cls) -> str: + """See :py:meth:`FilesystemInfo.fs_type` for documentation on this method.""" return "cephfs" def _get_or_create_auth_secret(self, model: Model) -> ops.Secret: @@ -505,25 +504,28 @@ def _get_or_create_auth_secret(self, model: Model) -> ops.Secret: class Endpoint: """Endpoint data exposed by a filesystem server.""" - fs_info: FsInfo + info: FilesystemInfo """Filesystem information required to mount this endpoint.""" uri: str """Raw URI exposed by this endpoint.""" + # Right now this is unused on the client, but having the raw uri + # available was useful on a previous version of the charm, so leaving + # this exposed just in case we need it in the future. -def _uri_to_fs_info(uri: str, model: Model) -> FsInfo: +def _uri_to_fs_info(uri: str, model: Model) -> FilesystemInfo: scheme = uri.split("://", maxsplit=1)[0] - if scheme == NfsInfo.fs_type(): + if scheme == NfsInfo.filesystem_type(): return NfsInfo.from_uri(uri, model) - elif scheme == CephfsInfo.fs_type(): + elif scheme == CephfsInfo.filesystem_type(): return CephfsInfo.from_uri(uri, model) else: - raise InterfacesError(f"unsupported filesystem type `{scheme}`") + raise FilesystemInfoError(f"unsupported filesystem type `{scheme}`") -class _MountEvent(RelationEvent): - """Base event for mount-related events.""" +class FilesystemEvent(RelationEvent): + """Base event for filesystem-related events.""" @property def endpoint(self) -> Optional[Endpoint]: @@ -533,19 +535,19 @@ def endpoint(self) -> Optional[Endpoint]: return Endpoint(_uri_to_fs_info(uri, self.framework.model), uri) -class MountFsEvent(_MountEvent): +class MountFilesystemEvent(FilesystemEvent): """Emit when a filesystem is ready to be mounted.""" -class UmountFsEvent(_MountEvent): +class UmountFilesystemEvent(FilesystemEvent): """Emit when a filesystem needs to be unmounted.""" -class _FsRequiresEvents(CharmEvents): +class FilesystemRequiresEvents(CharmEvents): """Events that FS servers can emit.""" - mount_fs = EventSource(MountFsEvent) - umount_fs = EventSource(UmountFsEvent) + mount_filesystem = EventSource(MountFilesystemEvent) + umount_filesystem = EventSource(UmountFilesystemEvent) class _BaseInterface(Object): @@ -572,10 +574,10 @@ def relations(self) -> List[Relation]: return result -class FsRequires(_BaseInterface): +class FilesystemRequires(_BaseInterface): """Consumer-side interface of filesystem integrations.""" - on = _FsRequiresEvents() + on = FilesystemRequiresEvents() def __init__(self, charm: CharmBase, relation_name: str) -> None: super().__init__(charm, relation_name) @@ -586,13 +588,13 @@ def __init__(self, charm: CharmBase, relation_name: str) -> None: def _on_relation_changed(self, event: RelationChangedEvent) -> None: """Handle when the databag between client and server has been updated.""" - _logger.debug("Emitting `MountShare` event from `RelationChanged` hook") - self.on.mount_fs.emit(event.relation, app=event.app, unit=event.unit) + _logger.debug("emitting `MountFilesystem` event from `RelationChanged` hook") + self.on.mount_filesystem.emit(event.relation, app=event.app, unit=event.unit) def _on_relation_departed(self, event: RelationDepartedEvent) -> None: """Handle when server departs integration.""" - _logger.debug("Emitting `UmountShare` event from `RelationDeparted` hook") - self.on.umount_fs.emit(event.relation, app=event.app, unit=event.unit) + _logger.debug("emitting `UmountFilesystem` event from `RelationDeparted` hook") + self.on.umount_filesystem.emit(event.relation, app=event.app, unit=event.unit) @property def endpoints(self) -> List[Endpoint]: @@ -601,11 +603,11 @@ def endpoints(self) -> List[Endpoint]: for relation in self.relations: if not (uri := relation.data[relation.app].get("endpoint")): continue - result.append(Endpoint(fs_info=_uri_to_fs_info(uri, self.model), uri=uri)) + result.append(Endpoint(info=_uri_to_fs_info(uri, self.model), uri=uri)) return result -class FsProvides(_BaseInterface): +class FilesystemProvides(_BaseInterface): """Provider-side interface of filesystem integrations.""" def __init__(self, charm: CharmBase, relation_name: str, peer_relation_name: str) -> None: @@ -613,11 +615,11 @@ def __init__(self, charm: CharmBase, relation_name: str, peer_relation_name: str self._peer_relation_name = peer_relation_name self.framework.observe(charm.on[relation_name].relation_joined, self._update_relation) - def set_fs_info(self, fs_info: FsInfo) -> None: + def set_info(self, info: FilesystemInfo) -> None: """Set information to mount a filesystem. Args: - fs_info: Information required to mount the filesystem. + info: Information required to mount the filesystem. Notes: Only the application leader unit can set the filesystem data. @@ -625,12 +627,12 @@ def set_fs_info(self, fs_info: FsInfo) -> None: if not self.unit.is_leader(): return - uri = fs_info.to_uri(self.model) + uri = info.to_uri(self.model) self._endpoint = uri for relation in self.relations: - fs_info.grant(self.model, relation) + info.grant(self.model, relation) relation.data[self.app]["endpoint"] = uri def _update_relation(self, event: RelationJoinedEvent) -> None: @@ -666,8 +668,8 @@ def _get_state(self, key: str) -> Optional[str]: def _set_state(self, key: str, data: str) -> None: """Insert a value into the global state.""" if not self._peers: - raise InterfacesError( - "Peer relation can only be accessed after the relation is established" + raise FilesystemInfoError( + "peer relation can only be accessed after the relation is established" ) self._peers.data[self.app][key] = data diff --git a/tests/integration/server/requirements.txt b/tests/integration/server/requirements.txt index dcb03f2..5a9cac8 100644 --- a/tests/integration/server/requirements.txt +++ b/tests/integration/server/requirements.txt @@ -1 +1,2 @@ -ops ~= 2.8 \ No newline at end of file +ops ~= 2.8 +tenacity ~= 9.0.0 diff --git a/tests/integration/server/src/charm.py b/tests/integration/server/src/charm.py index fb12494..a29d067 100755 --- a/tests/integration/server/src/charm.py +++ b/tests/integration/server/src/charm.py @@ -4,50 +4,166 @@ """Charm the application.""" +import json import logging +import os +import socket +import subprocess +import textwrap +from pathlib import Path import ops -from charms.filesystem_client.v0.interfaces import CephfsInfo, FsProvides, NfsInfo +from charms.filesystem_client.v0.filesystem_info import CephfsInfo, FilesystemProvides, NfsInfo +from tenacity import retry, stop_after_attempt, wait_exponential _logger = logging.getLogger(__name__) -NFS_INFO = NfsInfo(hostname="192.168.1.254", path="/srv", port=65535) -CEPHFS_INFO = CephfsInfo( - fsid="123456789-0abc-defg-hijk-lmnopqrstuvw", - name="filesystem", - path="/export", - monitor_hosts=[ - "192.168.1.1:6789", - "192.168.1.2:6789", - "192.168.1.3:6789", - ], - user="user", - key="R//appdqz4NP4Bxcc5XWrg==", -) +def _exec_commands(cmds: [[str]]) -> None: + for cmd in cmds: + _exec_command(cmd) + + +def _exec_command(cmd: [str]) -> str: + _logger.info(f"executing `{' '.join(cmd)}`") + env = os.environ.copy() + env["DEBIAN_FRONTEND"] = "noninteractive" + try: + output = subprocess.check_output(cmd, text=True, env=env) + except subprocess.CalledProcessError as e: + _logger.error(e.output) + raise Exception(f"failed to execute command `{' '.join(cmd)}` in instance") + _logger.info(output) + return output class FilesystemServerCharm(ops.CharmBase): - def __init__(self, framework: ops.Framework): + def __init__(self, framework: ops.Framework) -> None: super().__init__(framework) - self._fs_share = FsProvides(self, "fs-share", "server-peers") + self._filesystem = FilesystemProvides(self, "filesystem", "server-peers") framework.observe(self.on.start, self._on_start) - def _on_start(self, event: ops.StartEvent): + def _on_start(self, event: ops.StartEvent) -> None: """Handle start event.""" _logger.info(self.config.get("type")) typ = self.config["type"] if "nfs" == typ: - info = NFS_INFO + info = self._deploy_nfs() elif "cephfs" == typ: - info = CEPHFS_INFO + info = self._deploy_cephfs() else: raise ValueError("invalid filesystem type") - self._fs_share.set_fs_info(info) - _logger.info("set info") + self._filesystem.set_info(info) + self.unit.status = ops.ActiveStatus() - _logger.info("transitioned to active") + + def _deploy_nfs(self) -> NfsInfo: + _exec_commands( + [ + ["apt", "update", "-y"], + ["apt", "upgrade", "-y"], + ["apt", "install", "-y", "nfs-kernel-server"], + ] + ) + + exports = textwrap.dedent( + """ + /srv *(ro,sync,subtree_check) + /data *(rw,sync,no_subtree_check,no_root_squash) + """ + ).strip("\n") + _logger.info(f"uploading the following /etc/exports file:\n{exports}") + Path("/etc/exports").write_text(exports) + _logger.info("starting NFS server") + Path("/data").mkdir() + + _exec_commands([["exportfs", "-a"], ["systemctl", "restart", "nfs-kernel-server"]]) + + for i in range(3): + Path(f"/data/test-{i}").touch() + + self.unit.set_ports(111, 2049, ops.Port("udp", 111), ops.Port("udp", 2049)) + public_ip = socket.gethostbyname(socket.gethostname()) + + return NfsInfo(hostname=public_ip, port=2049, path="/data") + + def _deploy_cephfs(self) -> CephfsInfo: + fs_name = "filesystem" + fs_user = "fs-client" + fs_path = "/" + + @retry(wait=wait_exponential(max=6), stop=stop_after_attempt(20)) + def _mount_cephfs() -> None: + # Need to extract this into its own function to apply the tenacity decorator + # Wait until the cluster is ready to mount the filesystem. + status = json.loads(_exec_command(["microceph.ceph", "-s", "-f", "json"])) + if status["health"]["status"] != "HEALTH_OK": + raise Exception("CephFS is not available") + + _exec_command(["mount", "-t", "ceph", f"admin@.{fs_name}={fs_path}", "/mnt"]) + + _exec_commands( + [ + ["ln", "-s", "/bin/true"], + ["apt", "update", "-y"], + ["apt", "upgrade", "-y"], + ["apt", "install", "-y", "ceph-common"], + ["snap", "install", "microceph"], + ["microceph", "cluster", "bootstrap"], + ["microceph", "disk", "add", "loop,2G,3"], + ["microceph.ceph", "osd", "pool", "create", f"{fs_name}_data"], + ["microceph.ceph", "osd", "pool", "create", f"{fs_name}_metadata"], + ["microceph.ceph", "fs", "new", fs_name, f"{fs_name}_metadata", f"{fs_name}_data"], + [ + "microceph.ceph", + "fs", + "authorize", + fs_name, + f"client.{fs_user}", + fs_path, + "rw", + ], + # Need to generate the test files inside microceph itself. + [ + "ln", + "-sf", + "/var/snap/microceph/current/conf/ceph.client.admin.keyring", + "/etc/ceph/ceph.client.admin.keyring", + ], + [ + "ln", + "-sf", + "/var/snap/microceph/current/conf/ceph.keyring", + "/etc/ceph/ceph.keyring", + ], + ["ln", "-sf", "/var/snap/microceph/current/conf/ceph.conf", "/etc/ceph/ceph.conf"], + ] + ) + + _mount_cephfs() + + for i in range(3): + Path(f"/mnt/test-{i}").touch() + + self.unit.set_ports(3300, 6789) + + status = json.loads(_exec_command(["microceph.ceph", "-s", "-f", "json"])) + fsid = status["fsid"] + key = _exec_command(["microceph.ceph", "auth", "print-key", f"client.{fs_user}"]) + hostname = socket.gethostbyname(socket.gethostname()) + + return CephfsInfo( + fsid=fsid, + name=fs_name, + path=fs_path, + monitor_hosts=[ + hostname + ":3300", + hostname + ":6789", + ], + user=fs_user, + key=key, + ) if __name__ == "__main__": # pragma: nocover diff --git a/tests/integration/test_charm.py b/tests/integration/test_charm.py index f71712c..555031d 100644 --- a/tests/integration/test_charm.py +++ b/tests/integration/test_charm.py @@ -9,7 +9,7 @@ import juju import pytest import yaml -from charms.filesystem_client.v0.interfaces import CephfsInfo, NfsInfo +from charms.filesystem_client.v0.filesystem_info import CephfsInfo, NfsInfo from pytest_operator.plugin import OpsTest logger = logging.getLogger(__name__) @@ -56,7 +56,7 @@ async def test_build_and_deploy(ops_test: OpsTest): application_name=APP_NAME, num_units=0, config={ - "mountinfo": """ + "mounts": """ { "nfs": { "mountpoint": "/nfs", @@ -79,6 +79,7 @@ async def test_build_and_deploy(ops_test: OpsTest): config={ "type": "nfs", }, + constraints=juju.constraints.parse("virt-type=virtual-machine"), ), ops_test.model.deploy( server, @@ -86,6 +87,7 @@ async def test_build_and_deploy(ops_test: OpsTest): config={ "type": "cephfs", }, + constraints=juju.constraints.parse("virt-type=virtual-machine root-disk=50G mem=8G"), ), ops_test.model.wait_for_idle( apps=["nfs-server", "cephfs-server", "ubuntu"], @@ -100,59 +102,27 @@ async def test_build_and_deploy(ops_test: OpsTest): @pytest.mark.order(2) async def test_integrate(ops_test: OpsTest): await ops_test.model.integrate(f"{APP_NAME}:juju-info", "ubuntu:juju-info") - await ops_test.model.integrate(f"{APP_NAME}:fs-share", "nfs-server:fs-share") - await ops_test.model.integrate(f"{APP_NAME}:fs-share", "cephfs-server:fs-share") + await ops_test.model.integrate(f"{APP_NAME}:filesystem", "nfs-server:filesystem") + await ops_test.model.integrate(f"{APP_NAME}:filesystem", "cephfs-server:filesystem") await ops_test.model.wait_for_idle( apps=[APP_NAME, "ubuntu", "nfs-server", "cephfs-server"], status="active", timeout=1000 ) -async def check_autofs( - ops_test: OpsTest, - master_file: str, - autofs_file: str, - mountpoint: str, - opts: [str], - endpoint: str, -): - unit = ops_test.model.applications["ubuntu"].units[0] - master = (await unit.ssh(f"cat {master_file}")).split() - mount = (await unit.ssh(f"cat {autofs_file}")).split() - - assert autofs_file == master[1] - assert mountpoint == mount[0] - assert set(opts) == set(mount[1].removeprefix("-").split(",")) - assert endpoint == mount[2] - - @pytest.mark.order(3) async def test_nfs_files(ops_test: OpsTest): - await check_autofs( - ops_test=ops_test, - master_file="/etc/auto.master.d/nfs.autofs", - autofs_file="/etc/auto.nfs", - mountpoint="/nfs", - opts=["exec", "suid", "nodev", "ro", f"port={NFS_INFO.port}"], - endpoint=f"{NFS_INFO.hostname}:{NFS_INFO.path}", - ) + unit = ops_test.model.applications["ubuntu"].units[0] + result = (await unit.ssh("ls /nfs")).strip("\n") + assert "test-0" in result + assert "test-1" in result + assert "test-2" in result @pytest.mark.order(4) async def test_cephfs_files(ops_test: OpsTest): - await check_autofs( - ops_test=ops_test, - master_file="/etc/auto.master.d/cephfs.autofs", - autofs_file="/etc/auto.cephfs", - mountpoint="/cephfs", - opts=[ - "noexec", - "nosuid", - "dev", - "rw", - "fstype=ceph", - f"mon_addr={"/".join(CEPHFS_INFO.monitor_hosts)}", - f"secret={CEPHFS_INFO.key}", - ], - endpoint=f"{CEPHFS_INFO.user}@{CEPHFS_INFO.fsid}.{CEPHFS_INFO.name}={CEPHFS_INFO.path}", - ) + unit = ops_test.model.applications["ubuntu"].units[0] + result = (await unit.ssh("ls /cephfs")).strip("\n") + assert "test-0" in result + assert "test-1" in result + assert "test-2" in result diff --git a/tests/unit/test_interfaces.py b/tests/unit/test_interfaces.py index cf30162..077f3a0 100644 --- a/tests/unit/test_interfaces.py +++ b/tests/unit/test_interfaces.py @@ -5,14 +5,14 @@ """Test the interfaces charm library.""" import pytest -from charms.filesystem_client.v0.interfaces import ( - FsRequires, +from charms.filesystem_client.v0.filesystem_info import ( + FilesystemRequires, _hostinfo, ) from ops import CharmBase -FS_INTEGRATION_NAME = "fs-share" -FS_INTEGRATION_INTERFACE = "fs_share" +FS_INTEGRATION_NAME = "filesystem" +FS_INTEGRATION_INTERFACE = "filesystem_info" FS_CLIENT_METADATA = f""" name: fs-client requires: @@ -33,7 +33,7 @@ class FsClientCharm(CharmBase): def __init__(self, *args, **kwargs) -> None: super().__init__(*args, **kwargs) - self.requirer = FsRequires(self, FS_INTEGRATION_NAME) + self.requirer = FilesystemRequires(self, FS_INTEGRATION_NAME) self.framework.observe(self.requirer.on.mount_fs, lambda *_: None) self.framework.observe(self.requirer.on.umount_fs, lambda *_: None) diff --git a/uv.lock b/uv.lock index ce79325..d065cad 100644 --- a/uv.lock +++ b/uv.lock @@ -12,11 +12,11 @@ wheels = [ [[package]] name = "attrs" -version = "24.2.0" +version = "24.3.0" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/fc/0f/aafca9af9315aee06a89ffde799a10a582fe8de76c563ee80bbcdc08b3fb/attrs-24.2.0.tar.gz", hash = "sha256:5cfb1b9148b5b086569baec03f20d7b6bf3bcacc9a42bebf87ffaaca362f6346", size = 792678 } +sdist = { url = "https://files.pythonhosted.org/packages/48/c8/6260f8ccc11f0917360fc0da435c5c9c7504e3db174d5a12a1494887b045/attrs-24.3.0.tar.gz", hash = "sha256:8f5c07333d543103541ba7be0e2ce16eeee8130cb0b3f9238ab904ce1e85baff", size = 805984 } wheels = [ - { url = "https://files.pythonhosted.org/packages/6a/21/5b6702a7f963e95456c0de2d495f67bf5fd62840ac655dc451586d23d39a/attrs-24.2.0-py3-none-any.whl", hash = "sha256:81921eb96de3191c8258c199618104dd27ac608d9366f5e35d011eae1867ede2", size = 63001 }, + { url = "https://files.pythonhosted.org/packages/89/aa/ab0f7891a01eeb2d2e338ae8fecbe57fcebea1a24dbb64d45801bfab481d/attrs-24.3.0-py3-none-any.whl", hash = "sha256:ac96cd038792094f438ad1f6ff80837353805ac950cd2aa0e0625ef19850c308", size = 63397 }, ] [[package]] @@ -60,11 +60,11 @@ wheels = [ [[package]] name = "certifi" -version = "2024.8.30" +version = "2024.12.14" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/b0/ee/9b19140fe824b367c04c5e1b369942dd754c4c5462d5674002f75c4dedc1/certifi-2024.8.30.tar.gz", hash = "sha256:bec941d2aa8195e248a60b31ff9f0558284cf01a52591ceda73ea9afffd69fd9", size = 168507 } +sdist = { url = "https://files.pythonhosted.org/packages/0f/bd/1d41ee578ce09523c81a15426705dd20969f5abf006d1afe8aeff0dd776a/certifi-2024.12.14.tar.gz", hash = "sha256:b650d30f370c2b724812bee08008be0c4163b163ddaec3f2546c1caf65f191db", size = 166010 } wheels = [ - { url = "https://files.pythonhosted.org/packages/12/90/3c9ff0512038035f59d279fddeb79f5f1eccd8859f06d6163c58798b9487/certifi-2024.8.30-py3-none-any.whl", hash = "sha256:922820b53db7a7257ffbda3f597266d435245903d80737e34f8a45ff3e3230d8", size = 167321 }, + { url = "https://files.pythonhosted.org/packages/a5/32/8f6669fc4798494966bf446c8c4a162e0b5d893dff088afddf76414f70e1/certifi-2024.12.14-py3-none-any.whl", hash = "sha256:1275f7a45be9464efc1173084eaa30f866fe2e47d389406136d332ed4967ec56", size = 164927 }, ] [[package]] From c24f4111a6e9a30d3bb41da57ad50a432121219c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jos=C3=A9=20Juli=C3=A1n=20Espina?= Date: Tue, 17 Dec 2024 18:06:51 -0600 Subject: [PATCH 2/3] chore: avoid vendoring charm libs This avoids having big diffs when bumping the version of charm libs. --- .github/workflows/ci.yaml | 6 + .gitignore | 7 +- charmcraft.yaml | 6 + justfile | 20 +- lib/charms/operator_libs_linux/v0/apt.py | 1361 ------------------ lib/charms/operator_libs_linux/v1/systemd.py | 288 ---- 6 files changed, 32 insertions(+), 1656 deletions(-) delete mode 100644 lib/charms/operator_libs_linux/v0/apt.py delete mode 100644 lib/charms/operator_libs_linux/v1/systemd.py diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index af3b333..dba19a8 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -45,6 +45,8 @@ jobs: uses: astral-sh/setup-uv@v4 with: version: "0.5.8" + - name: Install Charmcraft + run: sudo snap install charmcraft --classic --channel latest/stable - name: Run linters run: just lint @@ -62,6 +64,8 @@ jobs: uses: astral-sh/setup-uv@v4 with: version: 0.5.8 + - name: Install Charmcraft + run: sudo snap install charmcraft --classic --channel latest/stable - name: Run unit tests run: just unit @@ -79,6 +83,8 @@ jobs: uses: astral-sh/setup-uv@v4 with: version: 0.5.8 + - name: Install Charmcraft + run: sudo snap install charmcraft --classic --channel latest/stable - name: Run tests run: just typecheck diff --git a/.gitignore b/.gitignore index 03da6e0..25ec1c7 100644 --- a/.gitignore +++ b/.gitignore @@ -7,4 +7,9 @@ __pycache__/ *.py[cod] .idea .vscode/ -requirements.txt \ No newline at end of file +requirements.txt + +# Ignore libs except the filesystem_client + +lib/charms/* +!lib/charms/filesystem_client/ diff --git a/charmcraft.yaml b/charmcraft.yaml index c2efd77..eb3df58 100644 --- a/charmcraft.yaml +++ b/charmcraft.yaml @@ -31,6 +31,12 @@ parts: just requirements craftctl default +charm-libs: + - lib: operator-libs-linux.apt + version: "0.15" + - lib: operator-libs-linux.systemd + version: "1.4" + subordinate: true requires: diff --git a/justfile b/justfile index 210924a..88808ed 100644 --- a/justfile +++ b/justfile @@ -1,4 +1,5 @@ uv := `which uv` +charmcraft := `which charmcraft` project_dir := justfile_directory() @@ -13,10 +14,18 @@ export PYTHONBREAKPOINT := "pdb.set_trace" uv_run := "uv run --frozen --extra dev" -# Regenerate uv.lock +# Regenerate uv.lock. lock: uv lock --no-cache +# Fetch the required charm libraries. +fetch-libs: + charmcraft fetch-libs + +# Create a development environment. +env: lock fetch-libs + uv sync --extra dev + # Upgrade uv.lock with the latest deps upgrade: uv lock --upgrade --no-cache @@ -27,23 +36,22 @@ requirements: lock # Apply coding style standards to code fmt: lock - echo {{PYTHONPATH}} {{uv_run}} ruff format {{all}} {{uv_run}} ruff check --fix {{all}} # Check code against coding style standards -lint: lock +lint: lock fetch-libs {{uv_run}} codespell {{lib}} {{uv_run}} codespell {{project_dir}} {{uv_run}} ruff check {{all}} {{uv_run}} ruff format --check --diff {{all}} # Run static type checks -typecheck *args: lock +typecheck *args: lock fetch-libs {{uv_run}} pyright {{args}} # Run unit tests -unit *args: lock +unit *args: lock fetch-libs {{uv_run}} coverage run \ --source={{src}} \ --source={{lib}} \ @@ -56,7 +64,7 @@ unit *args: lock {{uv_run}} coverage report # Run integration tests -integration *args: lock +integration *args: lock fetch-libs {{uv_run}} pytest \ -v \ -s \ diff --git a/lib/charms/operator_libs_linux/v0/apt.py b/lib/charms/operator_libs_linux/v0/apt.py deleted file mode 100644 index b8913c0..0000000 --- a/lib/charms/operator_libs_linux/v0/apt.py +++ /dev/null @@ -1,1361 +0,0 @@ -# Copyright 2021 Canonical Ltd. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""Abstractions for the system's Debian/Ubuntu package information and repositories. - -This module contains abstractions and wrappers around Debian/Ubuntu-style repositories and -packages, in order to easily provide an idiomatic and Pythonic mechanism for adding packages and/or -repositories to systems for use in machine charms. - -A sane default configuration is attainable through nothing more than instantiation of the -appropriate classes. `DebianPackage` objects provide information about the architecture, version, -name, and status of a package. - -`DebianPackage` will try to look up a package either from `dpkg -L` or from `apt-cache` when -provided with a string indicating the package name. If it cannot be located, `PackageNotFoundError` -will be returned, as `apt` and `dpkg` otherwise return `100` for all errors, and a meaningful error -message if the package is not known is desirable. - -To install packages with convenience methods: - -```python -try: - # Run `apt-get update` - apt.update() - apt.add_package("zsh") - apt.add_package(["vim", "htop", "wget"]) -except PackageNotFoundError: - logger.error("a specified package not found in package cache or on system") -except PackageError as e: - logger.error("could not install package. Reason: %s", e.message) -```` - -To find details of a specific package: - -```python -try: - vim = apt.DebianPackage.from_system("vim") - - # To find from the apt cache only - # apt.DebianPackage.from_apt_cache("vim") - - # To find from installed packages only - # apt.DebianPackage.from_installed_package("vim") - - vim.ensure(PackageState.Latest) - logger.info("updated vim to version: %s", vim.fullversion) -except PackageNotFoundError: - logger.error("a specified package not found in package cache or on system") -except PackageError as e: - logger.error("could not install package. Reason: %s", e.message) -``` - - -`RepositoryMapping` will return a dict-like object containing enabled system repositories -and their properties (available groups, baseuri. gpg key). This class can add, disable, or -manipulate repositories. Items can be retrieved as `DebianRepository` objects. - -In order add a new repository with explicit details for fields, a new `DebianRepository` can -be added to `RepositoryMapping` - -`RepositoryMapping` provides an abstraction around the existing repositories on the system, -and can be accessed and iterated over like any `Mapping` object, to retrieve values by key, -iterate, or perform other operations. - -Keys are constructed as `{repo_type}-{}-{release}` in order to uniquely identify a repository. - -Repositories can be added with explicit values through a Python constructor. - -Example: -```python -repositories = apt.RepositoryMapping() - -if "deb-example.com-focal" not in repositories: - repositories.add(DebianRepository(enabled=True, repotype="deb", - uri="https://example.com", release="focal", groups=["universe"])) -``` - -Alternatively, any valid `sources.list` line may be used to construct a new -`DebianRepository`. - -Example: -```python -repositories = apt.RepositoryMapping() - -if "deb-us.archive.ubuntu.com-xenial" not in repositories: - line = "deb http://us.archive.ubuntu.com/ubuntu xenial main restricted" - repo = DebianRepository.from_repo_line(line) - repositories.add(repo) -``` -""" - -import fileinput -import glob -import logging -import os -import re -import subprocess -from collections.abc import Mapping -from enum import Enum -from subprocess import PIPE, CalledProcessError, check_output -from typing import Iterable, List, Optional, Tuple, Union -from urllib.parse import urlparse - -logger = logging.getLogger(__name__) - -# The unique Charmhub library identifier, never change it -LIBID = "7c3dbc9c2ad44a47bd6fcb25caa270e5" - -# Increment this major API version when introducing breaking changes -LIBAPI = 0 - -# Increment this PATCH version before using `charmcraft publish-lib` or reset -# to 0 if you are raising the major API version -LIBPATCH = 14 - - -VALID_SOURCE_TYPES = ("deb", "deb-src") -OPTIONS_MATCHER = re.compile(r"\[.*?\]") - - -class Error(Exception): - """Base class of most errors raised by this library.""" - - def __repr__(self): - """Represent the Error.""" - return "<{}.{} {}>".format(type(self).__module__, type(self).__name__, self.args) - - @property - def name(self): - """Return a string representation of the model plus class.""" - return "<{}.{}>".format(type(self).__module__, type(self).__name__) - - @property - def message(self): - """Return the message passed as an argument.""" - return self.args[0] - - -class PackageError(Error): - """Raised when there's an error installing or removing a package.""" - - -class PackageNotFoundError(Error): - """Raised when a requested package is not known to the system.""" - - -class PackageState(Enum): - """A class to represent possible package states.""" - - Present = "present" - Absent = "absent" - Latest = "latest" - Available = "available" - - -class DebianPackage: - """Represents a traditional Debian package and its utility functions. - - `DebianPackage` wraps information and functionality around a known package, whether installed - or available. The version, epoch, name, and architecture can be easily queried and compared - against other `DebianPackage` objects to determine the latest version or to install a specific - version. - - The representation of this object as a string mimics the output from `dpkg` for familiarity. - - Installation and removal of packages is handled through the `state` property or `ensure` - method, with the following options: - - apt.PackageState.Absent - apt.PackageState.Available - apt.PackageState.Present - apt.PackageState.Latest - - When `DebianPackage` is initialized, the state of a given `DebianPackage` object will be set to - `Available`, `Present`, or `Latest`, with `Absent` implemented as a convenience for removal - (though it operates essentially the same as `Available`). - """ - - def __init__( - self, name: str, version: str, epoch: str, arch: str, state: PackageState - ) -> None: - self._name = name - self._arch = arch - self._state = state - self._version = Version(version, epoch) - - def __eq__(self, other) -> bool: - """Equality for comparison. - - Args: - other: a `DebianPackage` object for comparison - - Returns: - A boolean reflecting equality - """ - return isinstance(other, self.__class__) and ( - self._name, - self._version.number, - ) == (other._name, other._version.number) - - def __hash__(self): - """Return a hash of this package.""" - return hash((self._name, self._version.number)) - - def __repr__(self): - """Represent the package.""" - return "<{}.{}: {}>".format(self.__module__, self.__class__.__name__, self.__dict__) - - def __str__(self): - """Return a human-readable representation of the package.""" - return "<{}: {}-{}.{} -- {}>".format( - self.__class__.__name__, - self._name, - self._version, - self._arch, - str(self._state), - ) - - @staticmethod - def _apt( - command: str, - package_names: Union[str, List], - optargs: Optional[List[str]] = None, - ) -> None: - """Wrap package management commands for Debian/Ubuntu systems. - - Args: - command: the command given to `apt-get` - package_names: a package name or list of package names to operate on - optargs: an (Optional) list of additioanl arguments - - Raises: - PackageError if an error is encountered - """ - optargs = optargs if optargs is not None else [] - if isinstance(package_names, str): - package_names = [package_names] - _cmd = ["apt-get", "-y", *optargs, command, *package_names] - try: - env = os.environ.copy() - env["DEBIAN_FRONTEND"] = "noninteractive" - subprocess.run(_cmd, capture_output=True, check=True, text=True, env=env) - except CalledProcessError as e: - raise PackageError( - "Could not {} package(s) [{}]: {}".format(command, [*package_names], e.stderr) - ) from None - - def _add(self) -> None: - """Add a package to the system.""" - self._apt( - "install", - "{}={}".format(self.name, self.version), - optargs=["--option=Dpkg::Options::=--force-confold"], - ) - - def _remove(self) -> None: - """Remove a package from the system. Implementation-specific.""" - return self._apt("remove", "{}={}".format(self.name, self.version)) - - @property - def name(self) -> str: - """Returns the name of the package.""" - return self._name - - def ensure(self, state: PackageState): - """Ensure that a package is in a given state. - - Args: - state: a `PackageState` to reconcile the package to - - Raises: - PackageError from the underlying call to apt - """ - if self._state is not state: - if state not in (PackageState.Present, PackageState.Latest): - self._remove() - else: - self._add() - self._state = state - - @property - def present(self) -> bool: - """Returns whether or not a package is present.""" - return self._state in (PackageState.Present, PackageState.Latest) - - @property - def latest(self) -> bool: - """Returns whether the package is the most recent version.""" - return self._state is PackageState.Latest - - @property - def state(self) -> PackageState: - """Returns the current package state.""" - return self._state - - @state.setter - def state(self, state: PackageState) -> None: - """Set the package state to a given value. - - Args: - state: a `PackageState` to reconcile the package to - - Raises: - PackageError from the underlying call to apt - """ - if state in (PackageState.Latest, PackageState.Present): - self._add() - else: - self._remove() - self._state = state - - @property - def version(self) -> "Version": - """Returns the version for a package.""" - return self._version - - @property - def epoch(self) -> str: - """Returns the epoch for a package. May be unset.""" - return self._version.epoch - - @property - def arch(self) -> str: - """Returns the architecture for a package.""" - return self._arch - - @property - def fullversion(self) -> str: - """Returns the name+epoch for a package.""" - return "{}.{}".format(self._version, self._arch) - - @staticmethod - def _get_epoch_from_version(version: str) -> Tuple[str, str]: - """Pull the epoch, if any, out of a version string.""" - epoch_matcher = re.compile(r"^((?P\d+):)?(?P.*)") - matches = epoch_matcher.search(version).groupdict() - return matches.get("epoch", ""), matches.get("version") - - @classmethod - def from_system( - cls, package: str, version: Optional[str] = "", arch: Optional[str] = "" - ) -> "DebianPackage": - """Locates a package, either on the system or known to apt, and serializes the information. - - Args: - package: a string representing the package - version: an optional string if a specific version is requested - arch: an optional architecture, defaulting to `dpkg --print-architecture`. If an - architecture is not specified, this will be used for selection. - - """ - try: - return DebianPackage.from_installed_package(package, version, arch) - except PackageNotFoundError: - logger.debug( - "package '%s' is not currently installed or has the wrong architecture.", package - ) - - # Ok, try `apt-cache ...` - try: - return DebianPackage.from_apt_cache(package, version, arch) - except (PackageNotFoundError, PackageError): - # If we get here, it's not known to the systems. - # This seems unnecessary, but virtually all `apt` commands have a return code of `100`, - # and providing meaningful error messages without this is ugly. - raise PackageNotFoundError( - "Package '{}{}' could not be found on the system or in the apt cache!".format( - package, ".{}".format(arch) if arch else "" - ) - ) from None - - @classmethod - def from_installed_package( - cls, package: str, version: Optional[str] = "", arch: Optional[str] = "" - ) -> "DebianPackage": - """Check whether the package is already installed and return an instance. - - Args: - package: a string representing the package - version: an optional string if a specific version is requested - arch: an optional architecture, defaulting to `dpkg --print-architecture`. - If an architecture is not specified, this will be used for selection. - """ - system_arch = check_output( - ["dpkg", "--print-architecture"], universal_newlines=True - ).strip() - arch = arch if arch else system_arch - - # Regexps are a really terrible way to do this. Thanks dpkg - output = "" - try: - output = check_output(["dpkg", "-l", package], stderr=PIPE, universal_newlines=True) - except CalledProcessError: - raise PackageNotFoundError("Package is not installed: {}".format(package)) from None - - # Pop off the output from `dpkg -l' because there's no flag to - # omit it` - lines = str(output).splitlines()[5:] - - dpkg_matcher = re.compile( - r""" - ^(?P\w+?)\s+ - (?P.*?)(?P:\w+?)?\s+ - (?P.*?)\s+ - (?P\w+?)\s+ - (?P.*) - """, - re.VERBOSE, - ) - - for line in lines: - try: - matches = dpkg_matcher.search(line).groupdict() - package_status = matches["package_status"] - - if not package_status.endswith("i"): - logger.debug( - "package '%s' in dpkg output but not installed, status: '%s'", - package, - package_status, - ) - break - - epoch, split_version = DebianPackage._get_epoch_from_version(matches["version"]) - pkg = DebianPackage( - matches["package_name"], - split_version, - epoch, - matches["arch"], - PackageState.Present, - ) - if (pkg.arch == "all" or pkg.arch == arch) and ( - version == "" or str(pkg.version) == version - ): - return pkg - except AttributeError: - logger.warning("dpkg matcher could not parse line: %s", line) - - # If we didn't find it, fail through - raise PackageNotFoundError("Package {}.{} is not installed!".format(package, arch)) - - @classmethod - def from_apt_cache( - cls, package: str, version: Optional[str] = "", arch: Optional[str] = "" - ) -> "DebianPackage": - """Check whether the package is already installed and return an instance. - - Args: - package: a string representing the package - version: an optional string if a specific version is requested - arch: an optional architecture, defaulting to `dpkg --print-architecture`. - If an architecture is not specified, this will be used for selection. - """ - system_arch = check_output( - ["dpkg", "--print-architecture"], universal_newlines=True - ).strip() - arch = arch if arch else system_arch - - # Regexps are a really terrible way to do this. Thanks dpkg - keys = ("Package", "Architecture", "Version") - - try: - output = check_output( - ["apt-cache", "show", package], stderr=PIPE, universal_newlines=True - ) - except CalledProcessError as e: - raise PackageError( - "Could not list packages in apt-cache: {}".format(e.stderr) - ) from None - - pkg_groups = output.strip().split("\n\n") - keys = ("Package", "Architecture", "Version") - - for pkg_raw in pkg_groups: - lines = str(pkg_raw).splitlines() - vals = {} - for line in lines: - if line.startswith(keys): - items = line.split(":", 1) - vals[items[0]] = items[1].strip() - else: - continue - - epoch, split_version = DebianPackage._get_epoch_from_version(vals["Version"]) - pkg = DebianPackage( - vals["Package"], - split_version, - epoch, - vals["Architecture"], - PackageState.Available, - ) - - if (pkg.arch == "all" or pkg.arch == arch) and ( - version == "" or str(pkg.version) == version - ): - return pkg - - # If we didn't find it, fail through - raise PackageNotFoundError("Package {}.{} is not in the apt cache!".format(package, arch)) - - -class Version: - """An abstraction around package versions. - - This seems like it should be strictly unnecessary, except that `apt_pkg` is not usable inside a - venv, and wedging version comparisons into `DebianPackage` would overcomplicate it. - - This class implements the algorithm found here: - https://www.debian.org/doc/debian-policy/ch-controlfields.html#version - """ - - def __init__(self, version: str, epoch: str): - self._version = version - self._epoch = epoch or "" - - def __repr__(self): - """Represent the package.""" - return "<{}.{}: {}>".format(self.__module__, self.__class__.__name__, self.__dict__) - - def __str__(self): - """Return human-readable representation of the package.""" - return "{}{}".format("{}:".format(self._epoch) if self._epoch else "", self._version) - - @property - def epoch(self): - """Returns the epoch for a package. May be empty.""" - return self._epoch - - @property - def number(self) -> str: - """Returns the version number for a package.""" - return self._version - - def _get_parts(self, version: str) -> Tuple[str, str]: - """Separate the version into component upstream and Debian pieces.""" - try: - version.rindex("-") - except ValueError: - # No hyphens means no Debian version - return version, "0" - - upstream, debian = version.rsplit("-", 1) - return upstream, debian - - def _listify(self, revision: str) -> List[str]: - """Split a revision string into a listself. - - This list is comprised of alternating between strings and numbers, - padded on either end to always be "str, int, str, int..." and - always be of even length. This allows us to trivially implement the - comparison algorithm described. - """ - result = [] - while revision: - rev_1, remains = self._get_alphas(revision) - rev_2, remains = self._get_digits(remains) - result.extend([rev_1, rev_2]) - revision = remains - return result - - def _get_alphas(self, revision: str) -> Tuple[str, str]: - """Return a tuple of the first non-digit characters of a revision.""" - # get the index of the first digit - for i, char in enumerate(revision): - if char.isdigit(): - if i == 0: - return "", revision - return revision[0:i], revision[i:] - # string is entirely alphas - return revision, "" - - def _get_digits(self, revision: str) -> Tuple[int, str]: - """Return a tuple of the first integer characters of a revision.""" - # If the string is empty, return (0,'') - if not revision: - return 0, "" - # get the index of the first non-digit - for i, char in enumerate(revision): - if not char.isdigit(): - if i == 0: - return 0, revision - return int(revision[0:i]), revision[i:] - # string is entirely digits - return int(revision), "" - - def _dstringcmp(self, a, b): # noqa: C901 - """Debian package version string section lexical sort algorithm. - - The lexical comparison is a comparison of ASCII values modified so - that all the letters sort earlier than all the non-letters and so that - a tilde sorts before anything, even the end of a part. - """ - if a == b: - return 0 - try: - for i, char in enumerate(a): - if char == b[i]: - continue - # "a tilde sorts before anything, even the end of a part" - # (emptyness) - if char == "~": - return -1 - if b[i] == "~": - return 1 - # "all the letters sort earlier than all the non-letters" - if char.isalpha() and not b[i].isalpha(): - return -1 - if not char.isalpha() and b[i].isalpha(): - return 1 - # otherwise lexical sort - if ord(char) > ord(b[i]): - return 1 - if ord(char) < ord(b[i]): - return -1 - except IndexError: - # a is longer than b but otherwise equal, greater unless there are tildes - if char == "~": - return -1 - return 1 - # if we get here, a is shorter than b but otherwise equal, so check for tildes... - if b[len(a)] == "~": - return 1 - return -1 - - def _compare_revision_strings(self, first: str, second: str): # noqa: C901 - """Compare two debian revision strings.""" - if first == second: - return 0 - - # listify pads results so that we will always be comparing ints to ints - # and strings to strings (at least until we fall off the end of a list) - first_list = self._listify(first) - second_list = self._listify(second) - if first_list == second_list: - return 0 - try: - for i, item in enumerate(first_list): - # explicitly raise IndexError if we've fallen off the edge of list2 - if i >= len(second_list): - raise IndexError - # if the items are equal, next - if item == second_list[i]: - continue - # numeric comparison - if isinstance(item, int): - if item > second_list[i]: - return 1 - if item < second_list[i]: - return -1 - else: - # string comparison - return self._dstringcmp(item, second_list[i]) - except IndexError: - # rev1 is longer than rev2 but otherwise equal, hence greater - # ...except for goddamn tildes - if first_list[len(second_list)][0][0] == "~": - return 1 - return 1 - # rev1 is shorter than rev2 but otherwise equal, hence lesser - # ...except for goddamn tildes - if second_list[len(first_list)][0][0] == "~": - return -1 - return -1 - - def _compare_version(self, other) -> int: - if (self.number, self.epoch) == (other.number, other.epoch): - return 0 - - if self.epoch < other.epoch: - return -1 - if self.epoch > other.epoch: - return 1 - - # If none of these are true, follow the algorithm - upstream_version, debian_version = self._get_parts(self.number) - other_upstream_version, other_debian_version = self._get_parts(other.number) - - upstream_cmp = self._compare_revision_strings(upstream_version, other_upstream_version) - if upstream_cmp != 0: - return upstream_cmp - - debian_cmp = self._compare_revision_strings(debian_version, other_debian_version) - if debian_cmp != 0: - return debian_cmp - - return 0 - - def __lt__(self, other) -> bool: - """Less than magic method impl.""" - return self._compare_version(other) < 0 - - def __eq__(self, other) -> bool: - """Equality magic method impl.""" - return self._compare_version(other) == 0 - - def __gt__(self, other) -> bool: - """Greater than magic method impl.""" - return self._compare_version(other) > 0 - - def __le__(self, other) -> bool: - """Less than or equal to magic method impl.""" - return self.__eq__(other) or self.__lt__(other) - - def __ge__(self, other) -> bool: - """Greater than or equal to magic method impl.""" - return self.__gt__(other) or self.__eq__(other) - - def __ne__(self, other) -> bool: - """Not equal to magic method impl.""" - return not self.__eq__(other) - - -def add_package( - package_names: Union[str, List[str]], - version: Optional[str] = "", - arch: Optional[str] = "", - update_cache: Optional[bool] = False, -) -> Union[DebianPackage, List[DebianPackage]]: - """Add a package or list of packages to the system. - - Args: - package_names: single package name, or list of package names - name: the name(s) of the package(s) - version: an (Optional) version as a string. Defaults to the latest known - arch: an optional architecture for the package - update_cache: whether or not to run `apt-get update` prior to operating - - Raises: - TypeError if no package name is given, or explicit version is set for multiple packages - PackageNotFoundError if the package is not in the cache. - PackageError if packages fail to install - """ - cache_refreshed = False - if update_cache: - update() - cache_refreshed = True - - packages = {"success": [], "retry": [], "failed": []} - - package_names = [package_names] if isinstance(package_names, str) else package_names - if not package_names: - raise TypeError("Expected at least one package name to add, received zero!") - - if len(package_names) != 1 and version: - raise TypeError( - "Explicit version should not be set if more than one package is being added!" - ) - - for p in package_names: - pkg, success = _add(p, version, arch) - if success: - packages["success"].append(pkg) - else: - logger.warning("failed to locate and install/update '%s'", pkg) - packages["retry"].append(p) - - if packages["retry"] and not cache_refreshed: - logger.info("updating the apt-cache and retrying installation of failed packages.") - update() - - for p in packages["retry"]: - pkg, success = _add(p, version, arch) - if success: - packages["success"].append(pkg) - else: - packages["failed"].append(p) - - if packages["failed"]: - raise PackageError("Failed to install packages: {}".format(", ".join(packages["failed"]))) - - return packages["success"] if len(packages["success"]) > 1 else packages["success"][0] - - -def _add( - name: str, - version: Optional[str] = "", - arch: Optional[str] = "", -) -> Tuple[Union[DebianPackage, str], bool]: - """Add a package to the system. - - Args: - name: the name(s) of the package(s) - version: an (Optional) version as a string. Defaults to the latest known - arch: an optional architecture for the package - - Returns: a tuple of `DebianPackage` if found, or a :str: if it is not, and - a boolean indicating success - """ - try: - pkg = DebianPackage.from_system(name, version, arch) - pkg.ensure(state=PackageState.Present) - return pkg, True - except PackageNotFoundError: - return name, False - - -def remove_package( - package_names: Union[str, List[str]] -) -> Union[DebianPackage, List[DebianPackage]]: - """Remove package(s) from the system. - - Args: - package_names: the name of a package - - Raises: - PackageNotFoundError if the package is not found. - """ - packages = [] - - package_names = [package_names] if isinstance(package_names, str) else package_names - if not package_names: - raise TypeError("Expected at least one package name to add, received zero!") - - for p in package_names: - try: - pkg = DebianPackage.from_installed_package(p) - pkg.ensure(state=PackageState.Absent) - packages.append(pkg) - except PackageNotFoundError: - logger.info("package '%s' was requested for removal, but it was not installed.", p) - - # the list of packages will be empty when no package is removed - logger.debug("packages: '%s'", packages) - return packages[0] if len(packages) == 1 else packages - - -def update() -> None: - """Update the apt cache via `apt-get update`.""" - subprocess.run(["apt-get", "update", "--error-on=any"], capture_output=True, check=True) - - -def import_key(key: str) -> str: - """Import an ASCII Armor key. - - A Radix64 format keyid is also supported for backwards - compatibility. In this case Ubuntu keyserver will be - queried for a key via HTTPS by its keyid. This method - is less preferable because https proxy servers may - require traffic decryption which is equivalent to a - man-in-the-middle attack (a proxy server impersonates - keyserver TLS certificates and has to be explicitly - trusted by the system). - - Args: - key: A GPG key in ASCII armor format, including BEGIN - and END markers or a keyid. - - Returns: - The GPG key filename written. - - Raises: - GPGKeyError if the key could not be imported - """ - key = key.strip() - if "-" in key or "\n" in key: - # Send everything not obviously a keyid to GPG to import, as - # we trust its validation better than our own. eg. handling - # comments before the key. - logger.debug("PGP key found (looks like ASCII Armor format)") - if ( - "-----BEGIN PGP PUBLIC KEY BLOCK-----" in key - and "-----END PGP PUBLIC KEY BLOCK-----" in key - ): - logger.debug("Writing provided PGP key in the binary format") - key_bytes = key.encode("utf-8") - key_name = DebianRepository._get_keyid_by_gpg_key(key_bytes) - key_gpg = DebianRepository._dearmor_gpg_key(key_bytes) - gpg_key_filename = "/etc/apt/trusted.gpg.d/{}.gpg".format(key_name) - DebianRepository._write_apt_gpg_keyfile( - key_name=gpg_key_filename, key_material=key_gpg - ) - return gpg_key_filename - else: - raise GPGKeyError("ASCII armor markers missing from GPG key") - else: - logger.warning( - "PGP key found (looks like Radix64 format). " - "SECURELY importing PGP key from keyserver; " - "full key not provided." - ) - # as of bionic add-apt-repository uses curl with an HTTPS keyserver URL - # to retrieve GPG keys. `apt-key adv` command is deprecated as is - # apt-key in general as noted in its manpage. See lp:1433761 for more - # history. Instead, /etc/apt/trusted.gpg.d is used directly to drop - # gpg - key_asc = DebianRepository._get_key_by_keyid(key) - # write the key in GPG format so that apt-key list shows it - key_gpg = DebianRepository._dearmor_gpg_key(key_asc.encode("utf-8")) - gpg_key_filename = "/etc/apt/trusted.gpg.d/{}.gpg".format(key) - DebianRepository._write_apt_gpg_keyfile(key_name=gpg_key_filename, key_material=key_gpg) - return gpg_key_filename - - -class InvalidSourceError(Error): - """Exceptions for invalid source entries.""" - - -class GPGKeyError(Error): - """Exceptions for GPG keys.""" - - -class DebianRepository: - """An abstraction to represent a repository.""" - - def __init__( - self, - enabled: bool, - repotype: str, - uri: str, - release: str, - groups: List[str], - filename: Optional[str] = "", - gpg_key_filename: Optional[str] = "", - options: Optional[dict] = None, - ): - self._enabled = enabled - self._repotype = repotype - self._uri = uri - self._release = release - self._groups = groups - self._filename = filename - self._gpg_key_filename = gpg_key_filename - self._options = options - - @property - def enabled(self): - """Return whether or not the repository is enabled.""" - return self._enabled - - @property - def repotype(self): - """Return whether it is binary or source.""" - return self._repotype - - @property - def uri(self): - """Return the URI.""" - return self._uri - - @property - def release(self): - """Return which Debian/Ubuntu releases it is valid for.""" - return self._release - - @property - def groups(self): - """Return the enabled package groups.""" - return self._groups - - @property - def filename(self): - """Returns the filename for a repository.""" - return self._filename - - @filename.setter - def filename(self, fname: str) -> None: - """Set the filename used when a repo is written back to disk. - - Args: - fname: a filename to write the repository information to. - """ - if not fname.endswith(".list"): - raise InvalidSourceError("apt source filenames should end in .list!") - - self._filename = fname - - @property - def gpg_key(self): - """Returns the path to the GPG key for this repository.""" - return self._gpg_key_filename - - @property - def options(self): - """Returns any additional repo options which are set.""" - return self._options - - def make_options_string(self) -> str: - """Generate the complete options string for a a repository. - - Combining `gpg_key`, if set, and the rest of the options to find - a complex repo string. - """ - options = self._options if self._options else {} - if self._gpg_key_filename: - options["signed-by"] = self._gpg_key_filename - - return ( - "[{}] ".format(" ".join(["{}={}".format(k, v) for k, v in options.items()])) - if options - else "" - ) - - @staticmethod - def prefix_from_uri(uri: str) -> str: - """Get a repo list prefix from the uri, depending on whether a path is set.""" - uridetails = urlparse(uri) - path = ( - uridetails.path.lstrip("/").replace("/", "-") if uridetails.path else uridetails.netloc - ) - return "/etc/apt/sources.list.d/{}".format(path) - - @staticmethod - def from_repo_line(repo_line: str, write_file: Optional[bool] = True) -> "DebianRepository": - """Instantiate a new `DebianRepository` a `sources.list` entry line. - - Args: - repo_line: a string representing a repository entry - write_file: boolean to enable writing the new repo to disk - """ - repo = RepositoryMapping._parse(repo_line, "UserInput") - fname = "{}-{}.list".format( - DebianRepository.prefix_from_uri(repo.uri), repo.release.replace("/", "-") - ) - repo.filename = fname - - options = repo.options if repo.options else {} - if repo.gpg_key: - options["signed-by"] = repo.gpg_key - - # For Python 3.5 it's required to use sorted in the options dict in order to not have - # different results in the order of the options between executions. - options_str = ( - "[{}] ".format(" ".join(["{}={}".format(k, v) for k, v in sorted(options.items())])) - if options - else "" - ) - - if write_file: - with open(fname, "wb") as f: - f.write( - ( - "{}".format("#" if not repo.enabled else "") - + "{} {}{} ".format(repo.repotype, options_str, repo.uri) - + "{} {}\n".format(repo.release, " ".join(repo.groups)) - ).encode("utf-8") - ) - - return repo - - def disable(self) -> None: - """Remove this repository from consideration. - - Disable it instead of removing from the repository file. - """ - searcher = "{} {}{} {}".format( - self.repotype, self.make_options_string(), self.uri, self.release - ) - for line in fileinput.input(self._filename, inplace=True): - if re.match(r"^{}\s".format(re.escape(searcher)), line): - print("# {}".format(line), end="") - else: - print(line, end="") - - def import_key(self, key: str) -> None: - """Import an ASCII Armor key. - - A Radix64 format keyid is also supported for backwards - compatibility. In this case Ubuntu keyserver will be - queried for a key via HTTPS by its keyid. This method - is less preferable because https proxy servers may - require traffic decryption which is equivalent to a - man-in-the-middle attack (a proxy server impersonates - keyserver TLS certificates and has to be explicitly - trusted by the system). - - Args: - key: A GPG key in ASCII armor format, - including BEGIN and END markers or a keyid. - - Raises: - GPGKeyError if the key could not be imported - """ - self._gpg_key_filename = import_key(key) - - @staticmethod - def _get_keyid_by_gpg_key(key_material: bytes) -> str: - """Get a GPG key fingerprint by GPG key material. - - Gets a GPG key fingerprint (40-digit, 160-bit) by the ASCII armor-encoded - or binary GPG key material. Can be used, for example, to generate file - names for keys passed via charm options. - """ - # Use the same gpg command for both Xenial and Bionic - cmd = ["gpg", "--with-colons", "--with-fingerprint"] - ps = subprocess.run( - cmd, - stdout=PIPE, - stderr=PIPE, - input=key_material, - ) - out, err = ps.stdout.decode(), ps.stderr.decode() - if "gpg: no valid OpenPGP data found." in err: - raise GPGKeyError("Invalid GPG key material provided") - # from gnupg2 docs: fpr :: Fingerprint (fingerprint is in field 10) - return re.search(r"^fpr:{9}([0-9A-F]{40}):$", out, re.MULTILINE).group(1) - - @staticmethod - def _get_key_by_keyid(keyid: str) -> str: - """Get a key via HTTPS from the Ubuntu keyserver. - - Different key ID formats are supported by SKS keyservers (the longer ones - are more secure, see "dead beef attack" and https://evil32.com/). Since - HTTPS is used, if SSLBump-like HTTPS proxies are in place, they will - impersonate keyserver.ubuntu.com and generate a certificate with - keyserver.ubuntu.com in the CN field or in SubjAltName fields of a - certificate. If such proxy behavior is expected it is necessary to add the - CA certificate chain containing the intermediate CA of the SSLBump proxy to - every machine that this code runs on via ca-certs cloud-init directive (via - cloudinit-userdata model-config) or via other means (such as through a - custom charm option). Also note that DNS resolution for the hostname in a - URL is done at a proxy server - not at the client side. - 8-digit (32 bit) key ID - https://keyserver.ubuntu.com/pks/lookup?search=0x4652B4E6 - 16-digit (64 bit) key ID - https://keyserver.ubuntu.com/pks/lookup?search=0x6E85A86E4652B4E6 - 40-digit key ID: - https://keyserver.ubuntu.com/pks/lookup?search=0x35F77D63B5CEC106C577ED856E85A86E4652B4E6 - - Args: - keyid: An 8, 16 or 40 hex digit keyid to find a key for - - Returns: - A string contining key material for the specified GPG key id - - - Raises: - subprocess.CalledProcessError - """ - # options=mr - machine-readable output (disables html wrappers) - keyserver_url = ( - "https://keyserver.ubuntu.com" "/pks/lookup?op=get&options=mr&exact=on&search=0x{}" - ) - curl_cmd = ["curl", keyserver_url.format(keyid)] - # use proxy server settings in order to retrieve the key - return check_output(curl_cmd).decode() - - @staticmethod - def _dearmor_gpg_key(key_asc: bytes) -> bytes: - """Convert a GPG key in the ASCII armor format to the binary format. - - Args: - key_asc: A GPG key in ASCII armor format. - - Returns: - A GPG key in binary format as a string - - Raises: - GPGKeyError - """ - ps = subprocess.run(["gpg", "--dearmor"], stdout=PIPE, stderr=PIPE, input=key_asc) - out, err = ps.stdout, ps.stderr.decode() - if "gpg: no valid OpenPGP data found." in err: - raise GPGKeyError( - "Invalid GPG key material. Check your network setup" - " (MTU, routing, DNS) and/or proxy server settings" - " as well as destination keyserver status." - ) - else: - return out - - @staticmethod - def _write_apt_gpg_keyfile(key_name: str, key_material: bytes) -> None: - """Write GPG key material into a file at a provided path. - - Args: - key_name: A key name to use for a key file (could be a fingerprint) - key_material: A GPG key material (binary) - """ - with open(key_name, "wb") as keyf: - keyf.write(key_material) - - -class RepositoryMapping(Mapping): - """An representation of known repositories. - - Instantiation of `RepositoryMapping` will iterate through the - filesystem, parse out repository files in `/etc/apt/...`, and create - `DebianRepository` objects in this list. - - Typical usage: - - repositories = apt.RepositoryMapping() - repositories.add(DebianRepository( - enabled=True, repotype="deb", uri="https://example.com", release="focal", - groups=["universe"] - )) - """ - - def __init__(self): - self._repository_map = {} - # Repositories that we're adding -- used to implement mode param - self.default_file = "/etc/apt/sources.list" - - # read sources.list if it exists - if os.path.isfile(self.default_file): - self.load(self.default_file) - - # read sources.list.d - for file in glob.iglob("/etc/apt/sources.list.d/*.list"): - self.load(file) - - def __contains__(self, key: str) -> bool: - """Magic method for checking presence of repo in mapping.""" - return key in self._repository_map - - def __len__(self) -> int: - """Return number of repositories in map.""" - return len(self._repository_map) - - def __iter__(self) -> Iterable[DebianRepository]: - """Return iterator for RepositoryMapping.""" - return iter(self._repository_map.values()) - - def __getitem__(self, repository_uri: str) -> DebianRepository: - """Return a given `DebianRepository`.""" - return self._repository_map[repository_uri] - - def __setitem__(self, repository_uri: str, repository: DebianRepository) -> None: - """Add a `DebianRepository` to the cache.""" - self._repository_map[repository_uri] = repository - - def load(self, filename: str): - """Load a repository source file into the cache. - - Args: - filename: the path to the repository file - """ - parsed = [] - skipped = [] - with open(filename, "r") as f: - for n, line in enumerate(f): - try: - repo = self._parse(line, filename) - except InvalidSourceError: - skipped.append(n) - else: - repo_identifier = "{}-{}-{}".format(repo.repotype, repo.uri, repo.release) - self._repository_map[repo_identifier] = repo - parsed.append(n) - logger.debug("parsed repo: '%s'", repo_identifier) - - if skipped: - skip_list = ", ".join(str(s) for s in skipped) - logger.debug("skipped the following lines in file '%s': %s", filename, skip_list) - - if parsed: - logger.info("parsed %d apt package repositories", len(parsed)) - else: - raise InvalidSourceError("all repository lines in '{}' were invalid!".format(filename)) - - @staticmethod - def _parse(line: str, filename: str) -> DebianRepository: - """Parse a line in a sources.list file. - - Args: - line: a single line from `load` to parse - filename: the filename being read - - Raises: - InvalidSourceError if the source type is unknown - """ - enabled = True - repotype = uri = release = gpg_key = "" - options = {} - groups = [] - - line = line.strip() - if line.startswith("#"): - enabled = False - line = line[1:] - - # Check for "#" in the line and treat a part after it as a comment then strip it off. - i = line.find("#") - if i > 0: - line = line[:i] - - # Split a source into substrings to initialize a new repo. - source = line.strip() - if source: - # Match any repo options, and get a dict representation. - for v in re.findall(OPTIONS_MATCHER, source): - opts = dict(o.split("=") for o in v.strip("[]").split()) - # Extract the 'signed-by' option for the gpg_key - gpg_key = opts.pop("signed-by", "") - options = opts - - # Remove any options from the source string and split the string into chunks - source = re.sub(OPTIONS_MATCHER, "", source) - chunks = source.split() - - # Check we've got a valid list of chunks - if len(chunks) < 3 or chunks[0] not in VALID_SOURCE_TYPES: - raise InvalidSourceError("An invalid sources line was found in %s!", filename) - - repotype = chunks[0] - uri = chunks[1] - release = chunks[2] - groups = chunks[3:] - - return DebianRepository( - enabled, repotype, uri, release, groups, filename, gpg_key, options - ) - else: - raise InvalidSourceError("An invalid sources line was found in %s!", filename) - - def add(self, repo: DebianRepository, default_filename: Optional[bool] = False) -> None: - """Add a new repository to the system. - - Args: - repo: a `DebianRepository` object - default_filename: an (Optional) filename if the default is not desirable - """ - new_filename = "{}-{}.list".format( - DebianRepository.prefix_from_uri(repo.uri), repo.release.replace("/", "-") - ) - - fname = repo.filename or new_filename - - options = repo.options if repo.options else {} - if repo.gpg_key: - options["signed-by"] = repo.gpg_key - - with open(fname, "wb") as f: - f.write( - ( - "{}".format("#" if not repo.enabled else "") - + "{} {}{} ".format(repo.repotype, repo.make_options_string(), repo.uri) - + "{} {}\n".format(repo.release, " ".join(repo.groups)) - ).encode("utf-8") - ) - - self._repository_map["{}-{}-{}".format(repo.repotype, repo.uri, repo.release)] = repo - - def disable(self, repo: DebianRepository) -> None: - """Remove a repository. Disable by default. - - Args: - repo: a `DebianRepository` to disable - """ - searcher = "{} {}{} {}".format( - repo.repotype, repo.make_options_string(), repo.uri, repo.release - ) - - for line in fileinput.input(repo.filename, inplace=True): - if re.match(r"^{}\s".format(re.escape(searcher)), line): - print("# {}".format(line), end="") - else: - print(line, end="") - - self._repository_map["{}-{}-{}".format(repo.repotype, repo.uri, repo.release)] = repo diff --git a/lib/charms/operator_libs_linux/v1/systemd.py b/lib/charms/operator_libs_linux/v1/systemd.py deleted file mode 100644 index cdcbad6..0000000 --- a/lib/charms/operator_libs_linux/v1/systemd.py +++ /dev/null @@ -1,288 +0,0 @@ -# Copyright 2021 Canonical Ltd. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - - -"""Abstractions for stopping, starting and managing system services via systemd. - -This library assumes that your charm is running on a platform that uses systemd. E.g., -Centos 7 or later, Ubuntu Xenial (16.04) or later. - -For the most part, we transparently provide an interface to a commonly used selection of -systemd commands, with a few shortcuts baked in. For example, service_pause and -service_resume with run the mask/unmask and enable/disable invocations. - -Example usage: - -```python -from charms.operator_libs_linux.v0.systemd import service_running, service_reload - -# Start a service -if not service_running("mysql"): - success = service_start("mysql") - -# Attempt to reload a service, restarting if necessary -success = service_reload("nginx", restart_on_failure=True) -``` -""" - -__all__ = [ # Don't export `_systemctl`. (It's not the intended way of using this lib.) - "SystemdError", - "daemon_reload", - "service_disable", - "service_enable", - "service_failed", - "service_pause", - "service_reload", - "service_restart", - "service_resume", - "service_running", - "service_start", - "service_stop", -] - -import logging -import subprocess - -logger = logging.getLogger(__name__) - -# The unique Charmhub library identifier, never change it -LIBID = "045b0d179f6b4514a8bb9b48aee9ebaf" - -# Increment this major API version when introducing breaking changes -LIBAPI = 1 - -# Increment this PATCH version before using `charmcraft publish-lib` or reset -# to 0 if you are raising the major API version -LIBPATCH = 4 - - -class SystemdError(Exception): - """Custom exception for SystemD related errors.""" - - -def _systemctl(*args: str, check: bool = False) -> int: - """Control a system service using systemctl. - - Args: - *args: Arguments to pass to systemctl. - check: Check the output of the systemctl command. Default: False. - - Returns: - Returncode of systemctl command execution. - - Raises: - SystemdError: Raised if calling systemctl returns a non-zero returncode and check is True. - """ - cmd = ["systemctl", *args] - logger.debug(f"Executing command: {cmd}") - try: - proc = subprocess.run( - cmd, - stdout=subprocess.PIPE, - stderr=subprocess.STDOUT, - text=True, - bufsize=1, - encoding="utf-8", - check=check, - ) - logger.debug( - f"Command {cmd} exit code: {proc.returncode}. systemctl output:\n{proc.stdout}" - ) - return proc.returncode - except subprocess.CalledProcessError as e: - raise SystemdError( - f"Command {cmd} failed with returncode {e.returncode}. systemctl output:\n{e.stdout}" - ) - - -def service_running(service_name: str) -> bool: - """Report whether a system service is running. - - Args: - service_name: The name of the service to check. - - Return: - True if service is running/active; False if not. - """ - # If returncode is 0, this means that is service is active. - return _systemctl("--quiet", "is-active", service_name) == 0 - - -def service_failed(service_name: str) -> bool: - """Report whether a system service has failed. - - Args: - service_name: The name of the service to check. - - Returns: - True if service is marked as failed; False if not. - """ - # If returncode is 0, this means that the service has failed. - return _systemctl("--quiet", "is-failed", service_name) == 0 - - -def service_start(*args: str) -> bool: - """Start a system service. - - Args: - *args: Arguments to pass to `systemctl start` (normally the service name). - - Returns: - On success, this function returns True for historical reasons. - - Raises: - SystemdError: Raised if `systemctl start ...` returns a non-zero returncode. - """ - return _systemctl("start", *args, check=True) == 0 - - -def service_stop(*args: str) -> bool: - """Stop a system service. - - Args: - *args: Arguments to pass to `systemctl stop` (normally the service name). - - Returns: - On success, this function returns True for historical reasons. - - Raises: - SystemdError: Raised if `systemctl stop ...` returns a non-zero returncode. - """ - return _systemctl("stop", *args, check=True) == 0 - - -def service_restart(*args: str) -> bool: - """Restart a system service. - - Args: - *args: Arguments to pass to `systemctl restart` (normally the service name). - - Returns: - On success, this function returns True for historical reasons. - - Raises: - SystemdError: Raised if `systemctl restart ...` returns a non-zero returncode. - """ - return _systemctl("restart", *args, check=True) == 0 - - -def service_enable(*args: str) -> bool: - """Enable a system service. - - Args: - *args: Arguments to pass to `systemctl enable` (normally the service name). - - Returns: - On success, this function returns True for historical reasons. - - Raises: - SystemdError: Raised if `systemctl enable ...` returns a non-zero returncode. - """ - return _systemctl("enable", *args, check=True) == 0 - - -def service_disable(*args: str) -> bool: - """Disable a system service. - - Args: - *args: Arguments to pass to `systemctl disable` (normally the service name). - - Returns: - On success, this function returns True for historical reasons. - - Raises: - SystemdError: Raised if `systemctl disable ...` returns a non-zero returncode. - """ - return _systemctl("disable", *args, check=True) == 0 - - -def service_reload(service_name: str, restart_on_failure: bool = False) -> bool: - """Reload a system service, optionally falling back to restart if reload fails. - - Args: - service_name: The name of the service to reload. - restart_on_failure: - Boolean indicating whether to fall back to a restart if the reload fails. - - Returns: - On success, this function returns True for historical reasons. - - Raises: - SystemdError: Raised if `systemctl reload|restart ...` returns a non-zero returncode. - """ - try: - return _systemctl("reload", service_name, check=True) == 0 - except SystemdError: - if restart_on_failure: - return service_restart(service_name) - else: - raise - - -def service_pause(service_name: str) -> bool: - """Pause a system service. - - Stops the service and prevents the service from starting again at boot. - - Args: - service_name: The name of the service to pause. - - Returns: - On success, this function returns True for historical reasons. - - Raises: - SystemdError: Raised if service is still running after being paused by systemctl. - """ - _systemctl("disable", "--now", service_name) - _systemctl("mask", service_name) - - if service_running(service_name): - raise SystemdError(f"Attempted to pause {service_name!r}, but it is still running.") - - return True - - -def service_resume(service_name: str) -> bool: - """Resume a system service. - - Re-enable starting the service again at boot. Start the service. - - Args: - service_name: The name of the service to resume. - - Returns: - On success, this function returns True for historical reasons. - - Raises: - SystemdError: Raised if service is not running after being resumed by systemctl. - """ - _systemctl("unmask", service_name) - _systemctl("enable", "--now", service_name) - - if not service_running(service_name): - raise SystemdError(f"Attempted to resume {service_name!r}, but it is not running.") - - return True - - -def daemon_reload() -> bool: - """Reload systemd manager configuration. - - Returns: - On success, this function returns True for historical reasons. - - Raises: - SystemdError: Raised if `systemctl daemon-reload` returns a non-zero returncode. - """ - return _systemctl("daemon-reload", check=True) == 0 From 56798cb3f0a5dada7d7ecf34c0c770ca221f89b5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jos=C3=A9=20Juli=C3=A1n=20Espina?= Date: Wed, 18 Dec 2024 19:43:29 -0600 Subject: [PATCH 3/3] feat: rework client to only allow a single provider --- .github/workflows/ci.yaml | 4 + charmcraft.yaml | 45 +++-- .../filesystem_client/v0/filesystem_info.py | 12 +- pyproject.toml | 4 +- src/charm.py | 121 +++++++------ src/utils/manager.py | 40 ++--- .../filesystem_client/v0/filesystem_info.py | 12 +- tests/integration/test_charm.py | 161 +++++++++++------- uv.lock | 96 ++--------- 9 files changed, 220 insertions(+), 275 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index dba19a8..b5d9989 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -19,6 +19,10 @@ on: branches: - main +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + jobs: inclusive-naming-check: name: Inclusive naming check diff --git a/charmcraft.yaml b/charmcraft.yaml index eb3df58..b628329 100644 --- a/charmcraft.yaml +++ b/charmcraft.yaml @@ -25,8 +25,6 @@ parts: build-snaps: - astral-uv charm-requirements: ["requirements.txt"] - charm-binary-python-packages: - - rpds_py ~= 0.22.3 override-build: | just requirements craftctl default @@ -42,32 +40,33 @@ subordinate: true requires: filesystem: interface: filesystem_info + limit: 1 juju-info: interface: juju-info scope: container config: options: - mounts: - default: "{}" + mountpoint: + description: Location to mount the filesystem on the machine. type: string + noexec: + default: false description: | - Information to mount filesystems on the machine. This is specified as a JSON object string. - Example usage: - ```bash - $ juju config filesystem-client \ - mounts=< None: from abc import ABC, abstractmethod from dataclasses import dataclass from ipaddress import AddressValueError, IPv6Address -from typing import List, Optional, TypeVar, Self +from typing import List, Optional, TypeVar from urllib.parse import parse_qs, quote, unquote, urlencode, urlparse, urlunsplit import ops @@ -354,9 +354,7 @@ def from_uri(cls, uri: str, _model: Model) -> "NfsInfo": info = _UriData.from_uri(uri) if info.scheme != cls.filesystem_type(): - raise ParseUriError( - "could not parse uri with incompatible scheme into `NfsInfo`" - ) + raise ParseUriError("could not parse uri with incompatible scheme into `NfsInfo`") path = info.path @@ -582,16 +580,14 @@ class FilesystemRequires(_BaseInterface): def __init__(self, charm: CharmBase, relation_name: str) -> None: super().__init__(charm, relation_name) self.framework.observe(charm.on[relation_name].relation_changed, self._on_relation_changed) - self.framework.observe( - charm.on[relation_name].relation_departed, self._on_relation_departed - ) + self.framework.observe(charm.on[relation_name].relation_broken, self._on_relation_broken) def _on_relation_changed(self, event: RelationChangedEvent) -> None: """Handle when the databag between client and server has been updated.""" _logger.debug("emitting `MountFilesystem` event from `RelationChanged` hook") self.on.mount_filesystem.emit(event.relation, app=event.app, unit=event.unit) - def _on_relation_departed(self, event: RelationDepartedEvent) -> None: + def _on_relation_broken(self, event: RelationDepartedEvent) -> None: """Handle when server departs integration.""" _logger.debug("emitting `UmountFilesystem` event from `RelationDeparted` hook") self.on.umount_filesystem.emit(event.relation, app=event.app, unit=event.unit) diff --git a/pyproject.toml b/pyproject.toml index 19b0bed..71008d7 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -3,9 +3,7 @@ name = "filesystem-client" version = "0.0" requires-python = "==3.12.*" dependencies = [ - "ops ~= 2.8", - "jsonschema ~= 4.23.0", - "rpds_py ~= 0.22.3" + "ops ~= 2.8" ] [project.optional-dependencies] diff --git a/src/charm.py b/src/charm.py index 93c0e5e..648ac34 100755 --- a/src/charm.py +++ b/src/charm.py @@ -4,42 +4,40 @@ """Charm for the filesystem client.""" -import json import logging -from collections import Counter +from dataclasses import dataclass import ops from charms.filesystem_client.v0.filesystem_info import FilesystemRequires -from jsonschema import ValidationError, validate from utils.manager import MountsManager -logger = logging.getLogger(__name__) - -CONFIG_SCHEMA = { - "$schema": "http://json-schema.org/draft-04/schema#", - "type": "object", - "additionalProperties": { - "type": "object", - "required": ["mountpoint"], - "properties": { - "mountpoint": {"type": "string"}, - "noexec": {"type": "boolean"}, - "nosuid": {"type": "boolean"}, - "nodev": {"type": "boolean"}, - "read-only": {"type": "boolean"}, - }, - }, -} +_logger = logging.getLogger(__name__) class StopCharmError(Exception): """Exception raised when a method needs to finish the execution of the charm code.""" - def __init__(self, status: ops.StatusBase): + def __init__(self, status: ops.StatusBase) -> None: self.status = status +@dataclass(frozen=True) +class CharmConfig: + """Configuration for the charm.""" + + mountpoint: str + """Location to mount the filesystem on the machine.""" + noexec: bool + """Block execution of binaries on the filesystem.""" + nosuid: bool + """Do not honor suid and sgid bits on the filesystem.""" + nodev: bool + """Blocking interpretation of character and/or block devices on the filesystem.""" + read_only: bool + """Mount filesystem as read-only.""" + + # Trying to use a delta charm (one method per event) proved to be a bit unwieldy, since # we would have to handle multiple updates at once: # - mount requests @@ -51,11 +49,11 @@ def __init__(self, status: ops.StatusBase): # mount requests. # # A holistic charm (one method for all events) was a lot easier to deal with, -# simplifying the code to handle all the multiple relations. +# simplifying the code to handle all the events. class FilesystemClientCharm(ops.CharmBase): """Charm the application.""" - def __init__(self, framework: ops.Framework): + def __init__(self, framework: ops.Framework) -> None: super().__init__(framework) self._filesystems = FilesystemRequires(self, "filesystem") @@ -66,71 +64,66 @@ def __init__(self, framework: ops.Framework): framework.observe(self._filesystems.on.mount_filesystem, self._handle_event) framework.observe(self._filesystems.on.umount_filesystem, self._handle_event) - def _handle_event(self, event: ops.EventBase) -> None: # noqa: C901 + def _handle_event(self, event: ops.EventBase) -> None: + """Handle a Juju event.""" try: self.unit.status = ops.MaintenanceStatus("Updating status.") + # CephFS is not supported on LXD containers. + if not self._mounts_manager.supported(): + self.unit.status = ops.BlockedStatus("Cannot mount filesystems on LXD containers.") + return + self._ensure_installed() config = self._get_config() self._mount_filesystems(config) except StopCharmError as e: # This was the cleanest way to ensure the inner methods can still return prematurely # when an error occurs. - self.app.status = e.status + self.unit.status = e.status return - self.unit.status = ops.ActiveStatus("Mounted filesystems.") + self.unit.status = ops.ActiveStatus(f"Mounted filesystem at `{config.mountpoint}`.") - def _ensure_installed(self): + def _ensure_installed(self) -> None: """Ensure the required packages are installed into the unit.""" if not self._mounts_manager.installed: self.unit.status = ops.MaintenanceStatus("Installing required packages.") self._mounts_manager.install() - def _get_config(self) -> dict[str, dict[str, str | bool]]: + def _get_config(self) -> CharmConfig: """Get and validate the configuration of the charm.""" - try: - config = json.loads(str(self.config.get("mounts", ""))) - validate(config, CONFIG_SCHEMA) - config: dict[str, dict[str, str | bool]] = config - for fs, opts in config.items(): - for opt in ["noexec", "nosuid", "nodev", "read-only"]: - opts[opt] = opts.get(opt, False) - return config - except (json.JSONDecodeError, ValidationError) as e: + if not (mountpoint := self.config.get("mountpoint")): + raise StopCharmError(ops.BlockedStatus("Missing `mountpoint` in config.")) + + return CharmConfig( + mountpoint=str(mountpoint), + noexec=bool(self.config.get("noexec")), + nosuid=bool(self.config.get("nosuid")), + nodev=bool(self.config.get("nodev")), + read_only=bool(self.config.get("read-only")), + ) + + def _mount_filesystems(self, config: CharmConfig) -> None: + """Mount the filesystem for the charm.""" + endpoints = self._filesystems.endpoints + if not endpoints: raise StopCharmError( - ops.BlockedStatus(f"invalid configuration for option `mounts`. reason:\n{e}") + ops.BlockedStatus("Waiting for an integration with a filesystem provider.") ) - def _mount_filesystems(self, config: dict[str, dict[str, str | bool]]): - """Mount all available filesystems for the charm.""" - endpoints = self._filesystems.endpoints - for fs_type, count in Counter( - [endpoint.info.filesystem_type() for endpoint in endpoints] - ).items(): - if count > 1: - raise StopCharmError( - ops.BlockedStatus(f"Too many relations for mount type `{fs_type}`.") - ) + # This is limited to 1 relation. + endpoint = endpoints[0] - self.unit.status = ops.MaintenanceStatus("Ensuring filesystems are mounted.") + self.unit.status = ops.MaintenanceStatus("Mounting filesystem.") with self._mounts_manager.mounts() as mounts: - for endpoint in endpoints: - fs_type = endpoint.info.filesystem_type() - if not (options := config.get(fs_type)): - raise StopCharmError( - ops.BlockedStatus(f"Missing configuration for mount type `{fs_type}`.") - ) - - mountpoint = str(options["mountpoint"]) - - opts = [] - opts.append("noexec" if options.get("noexec") else "exec") - opts.append("nosuid" if options.get("nosuid") else "suid") - opts.append("nodev" if options.get("nodev") else "dev") - opts.append("ro" if options.get("read-only") else "rw") - mounts.add(info=endpoint.info, mountpoint=mountpoint, options=opts) + opts = [] + opts.append("noexec" if config.noexec else "exec") + opts.append("nosuid" if config.nosuid else "suid") + opts.append("nodev" if config.nodev else "dev") + opts.append("ro" if config.read_only else "rw") + mounts.add(info=endpoint.info, mountpoint=config.mountpoint, options=opts) if __name__ == "__main__": # pragma: nocover diff --git a/src/utils/manager.py b/src/utils/manager.py index 3a9fe7a..a542b2d 100644 --- a/src/utils/manager.py +++ b/src/utils/manager.py @@ -8,9 +8,9 @@ import os import pathlib import subprocess +from collections.abc import Iterable from dataclasses import dataclass from ipaddress import AddressValueError, IPv6Address -from typing import Generator, List, Optional, Union import charms.operator_libs_linux.v0.apt as apt import charms.operator_libs_linux.v1.systemd as systemd @@ -24,16 +24,16 @@ class Error(Exception): """Raise if Storage client manager encounters an error.""" @property - def name(self): + def name(self) -> str: """Get a string representation of the error plus class name.""" return f"<{type(self).__module__}.{type(self).__name__}>" @property - def message(self): + def message(self) -> str: """Return the message passed as an argument.""" return self.args[0] - def __repr__(self): + def __repr__(self) -> str: """Return the string representation of the error.""" return f"<{type(self).__module__}.{type(self).__name__} {self.args}>" @@ -63,16 +63,14 @@ class _MountInfo: class Mounts: """Collection of mounts that need to be managed by the `MountsManager`.""" - _mounts: dict[str, _MountInfo] - - def __init__(self): - self._mounts = {} + def __init__(self) -> None: + self._mounts: dict[str, _MountInfo] = {} def add( self, info: FilesystemInfo, - mountpoint: Union[str, os.PathLike], - options: Optional[List[str]] = None, + mountpoint: str | os.PathLike, + options: list[str] | None = None, ) -> None: """Add a mount to the list of managed mounts. @@ -96,14 +94,14 @@ def add( class MountsManager: """Manager for mounted filesystems in the current system.""" - def __init__(self, charm: ops.CharmBase): + def __init__(self, charm: ops.CharmBase) -> None: # Lazily initialized self._pkgs = None self._master_file = pathlib.Path(f"/etc/auto.master.d/{charm.app.name}.autofs") self._autofs_file = pathlib.Path(f"/etc/auto.{charm.app.name}") @property - def _packages(self) -> List[apt.DebianPackage]: + def _packages(self) -> list[apt.DebianPackage]: """List of packages required by the client.""" if not self._pkgs: self._pkgs = [ @@ -119,12 +117,12 @@ def installed(self) -> bool: if not pkg.present: return False - if not self._master_file.exists or not self._autofs_file.exists: + if not self._master_file.exists() or not self._autofs_file.exists(): return False return True - def install(self): + def install(self) -> None: """Install the required mount packages. Raises: @@ -134,9 +132,7 @@ def install(self): for pkg in self._packages: pkg.ensure(apt.PackageState.Present) except (apt.PackageError, apt.PackageNotFoundError) as e: - _logger.error( - f"failed to change the state of the required packages. reason:\n{e.message}" - ) + _logger.error("failed to change the state of the required packages", exc_info=e) raise Error(e.message) try: @@ -144,7 +140,7 @@ def install(self): self._autofs_file.touch(mode=0o600) self._master_file.write_text(f"/- {self._autofs_file}") except IOError as e: - _logger.error(f"failed to create the required autofs files. reason:\n{e}") + _logger.error("failed to create the required autofs files", exc_info=e) raise Error("failed to create the required autofs files") def supported(self) -> bool: @@ -159,11 +155,11 @@ def supported(self) -> bool: else: return True except subprocess.CalledProcessError: - _logger.warning("Could not detect execution in virtualized environment") + _logger.warning("could not detect execution in virtualized environment") return True @contextlib.contextmanager - def mounts(self, force_mount=False) -> Generator[Mounts, None, None]: + def mounts(self, force_mount=False) -> Iterable[Mounts]: """Get the list of `Mounts` that need to be managed by the `MountsManager`. It will initially contain no mounts, and any mount that is added to @@ -195,9 +191,7 @@ def mounts(self, force_mount=False) -> Generator[Mounts, None, None]: self._autofs_file.write_text(new_autofs) systemd.service_reload("autofs", restart_on_failure=True) except systemd.SystemdError as e: - _logger.error(f"failed to mount filesystems. reason:\n{e}") - if "Operation not permitted" in str(e) and not self.supported(): - raise Error("mounting shares not supported on LXD containers") + _logger.error("failed to mount filesystems", exc_info=e) raise Error("failed to mount filesystems") diff --git a/tests/integration/server/lib/charms/filesystem_client/v0/filesystem_info.py b/tests/integration/server/lib/charms/filesystem_client/v0/filesystem_info.py index d0e1ea7..dbce045 100644 --- a/tests/integration/server/lib/charms/filesystem_client/v0/filesystem_info.py +++ b/tests/integration/server/lib/charms/filesystem_client/v0/filesystem_info.py @@ -100,7 +100,7 @@ def _on_start(self, event: ops.StartEvent) -> None: from abc import ABC, abstractmethod from dataclasses import dataclass from ipaddress import AddressValueError, IPv6Address -from typing import List, Optional, TypeVar, Self +from typing import List, Optional, TypeVar from urllib.parse import parse_qs, quote, unquote, urlencode, urlparse, urlunsplit import ops @@ -354,9 +354,7 @@ def from_uri(cls, uri: str, _model: Model) -> "NfsInfo": info = _UriData.from_uri(uri) if info.scheme != cls.filesystem_type(): - raise ParseUriError( - "could not parse uri with incompatible scheme into `NfsInfo`" - ) + raise ParseUriError("could not parse uri with incompatible scheme into `NfsInfo`") path = info.path @@ -582,16 +580,14 @@ class FilesystemRequires(_BaseInterface): def __init__(self, charm: CharmBase, relation_name: str) -> None: super().__init__(charm, relation_name) self.framework.observe(charm.on[relation_name].relation_changed, self._on_relation_changed) - self.framework.observe( - charm.on[relation_name].relation_departed, self._on_relation_departed - ) + self.framework.observe(charm.on[relation_name].relation_broken, self._on_relation_broken) def _on_relation_changed(self, event: RelationChangedEvent) -> None: """Handle when the databag between client and server has been updated.""" _logger.debug("emitting `MountFilesystem` event from `RelationChanged` hook") self.on.mount_filesystem.emit(event.relation, app=event.app, unit=event.unit) - def _on_relation_departed(self, event: RelationDepartedEvent) -> None: + def _on_relation_broken(self, event: RelationDepartedEvent) -> None: """Handle when server departs integration.""" _logger.debug("emitting `UmountFilesystem` event from `RelationDeparted` hook") self.on.umount_filesystem.emit(event.relation, app=event.app, unit=event.unit) diff --git a/tests/integration/test_charm.py b/tests/integration/test_charm.py index 555031d..7ccdfb8 100644 --- a/tests/integration/test_charm.py +++ b/tests/integration/test_charm.py @@ -34,7 +34,7 @@ @pytest.mark.abort_on_fail @pytest.mark.order(1) -async def test_build_and_deploy(ops_test: OpsTest): +async def test_build_and_deploy(ops_test: OpsTest) -> None: """Build the charm-under-test and deploy it together with related charms. Assert on the unit status before any relations/configurations take place. @@ -44,74 +44,87 @@ async def test_build_and_deploy(ops_test: OpsTest): server = await ops_test.build_charm("./tests/integration/server") # Deploy the charm and wait for active/idle status - await asyncio.gather( - ops_test.model.deploy( - "ubuntu", - application_name="ubuntu", - base="ubuntu@24.04", - constraints=juju.constraints.parse("virt-type=virtual-machine"), - ), - ops_test.model.deploy( - charm, - application_name=APP_NAME, - num_units=0, - config={ - "mounts": """ - { - "nfs": { - "mountpoint": "/nfs", - "nodev": true, - "read-only": true + async with asyncio.TaskGroup() as tg: + tg.create_task( + ops_test.model.deploy( + "ubuntu", + application_name="ubuntu", + base="ubuntu@24.04", + constraints=juju.constraints.parse("virt-type=virtual-machine"), + ) + ) + tg.create_task( + ops_test.model.deploy( + charm, + application_name=APP_NAME, + num_units=0, + ) + ) + tg.create_task( + ops_test.model.deploy( + server, + application_name="nfs-server", + config={ + "type": "nfs", }, - "cephfs": { - "mountpoint": "/cephfs", - "noexec": true, - "nosuid": true, - "nodev": false - } - } - """ - }, - ), - ops_test.model.deploy( - server, - application_name="nfs-server", - config={ - "type": "nfs", - }, - constraints=juju.constraints.parse("virt-type=virtual-machine"), - ), - ops_test.model.deploy( - server, - application_name="cephfs-server", - config={ - "type": "cephfs", - }, - constraints=juju.constraints.parse("virt-type=virtual-machine root-disk=50G mem=8G"), - ), - ops_test.model.wait_for_idle( - apps=["nfs-server", "cephfs-server", "ubuntu"], - status="active", - raise_on_blocked=True, - timeout=1000, - ), - ) + constraints=juju.constraints.parse("virt-type=virtual-machine"), + ) + ) + tg.create_task( + ops_test.model.deploy( + server, + application_name="cephfs-server", + config={ + "type": "cephfs", + }, + constraints=juju.constraints.parse( + "virt-type=virtual-machine root-disk=50G mem=8G" + ), + ) + ) + tg.create_task( + ops_test.model.wait_for_idle( + apps=["nfs-server", "cephfs-server", "ubuntu"], + status="active", + raise_on_blocked=True, + raise_on_error=True, + timeout=1000, + ) + ) @pytest.mark.abort_on_fail @pytest.mark.order(2) -async def test_integrate(ops_test: OpsTest): - await ops_test.model.integrate(f"{APP_NAME}:juju-info", "ubuntu:juju-info") - await ops_test.model.integrate(f"{APP_NAME}:filesystem", "nfs-server:filesystem") - await ops_test.model.integrate(f"{APP_NAME}:filesystem", "cephfs-server:filesystem") - - await ops_test.model.wait_for_idle( - apps=[APP_NAME, "ubuntu", "nfs-server", "cephfs-server"], status="active", timeout=1000 +async def test_integrate(ops_test: OpsTest) -> None: + async with asyncio.TaskGroup() as tg: + tg.create_task(ops_test.model.integrate(f"{APP_NAME}:juju-info", "ubuntu:juju-info")) + tg.create_task( + ops_test.model.wait_for_idle(apps=["ubuntu"], status="active", raise_on_error=True) + ) + tg.create_task( + ops_test.model.wait_for_idle(apps=[APP_NAME], status="blocked", raise_on_error=True) + ) + + assert ( + ops_test.model.applications[APP_NAME].units[0].workload_status_message + == "Missing `mountpoint` in config." ) +@pytest.mark.abort_on_fail @pytest.mark.order(3) -async def test_nfs_files(ops_test: OpsTest): +async def test_nfs(ops_test: OpsTest) -> None: + async with asyncio.TaskGroup() as tg: + tg.create_task(ops_test.model.integrate(f"{APP_NAME}:filesystem", "nfs-server:filesystem")) + tg.create_task( + ops_test.model.applications[APP_NAME].set_config( + {"mountpoint": "/nfs", "nodev": "true", "read-only": "true"} + ) + ) + tg.create_task( + ops_test.model.wait_for_idle(apps=[APP_NAME], status="active", raise_on_error=True) + ) + unit = ops_test.model.applications["ubuntu"].units[0] result = (await unit.ssh("ls /nfs")).strip("\n") assert "test-0" in result @@ -119,8 +132,34 @@ async def test_nfs_files(ops_test: OpsTest): assert "test-2" in result +@pytest.mark.abort_on_fail @pytest.mark.order(4) -async def test_cephfs_files(ops_test: OpsTest): +async def test_cephfs(ops_test: OpsTest) -> None: + # Ensure the relation is removed before the config changes. + # This guarantees that the new mountpoint is fresh. + async with asyncio.TaskGroup() as tg: + tg.create_task( + ops_test.model.applications[APP_NAME].remove_relation( + "filesystem", "nfs-server:filesystem" + ) + ) + tg.create_task( + ops_test.model.wait_for_idle(apps=[APP_NAME], status="blocked", raise_on_error=True) + ) + + async with asyncio.TaskGroup() as tg: + tg.create_task( + ops_test.model.applications[APP_NAME].set_config( + {"mountpoint": "/cephfs", "noexec": "true", "nosuid": "true", "nodev": "false"} + ) + ) + tg.create_task( + ops_test.model.integrate(f"{APP_NAME}:filesystem", "cephfs-server:filesystem") + ) + tg.create_task( + ops_test.model.wait_for_idle(apps=[APP_NAME], status="active", raise_on_error=True) + ) + unit = ops_test.model.applications["ubuntu"].units[0] result = (await unit.ssh("ls /cephfs")).strip("\n") assert "test-0" in result diff --git a/uv.lock b/uv.lock index d065cad..7e10692 100644 --- a/uv.lock +++ b/uv.lock @@ -10,15 +10,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/25/8a/c46dcc25341b5bce5472c718902eb3d38600a903b14fa6aeecef3f21a46f/asttokens-3.0.0-py3-none-any.whl", hash = "sha256:e3078351a059199dd5138cb1c706e6430c05eff2ff136af5eb4790f9d28932e2", size = 26918 }, ] -[[package]] -name = "attrs" -version = "24.3.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/48/c8/6260f8ccc11f0917360fc0da435c5c9c7504e3db174d5a12a1494887b045/attrs-24.3.0.tar.gz", hash = "sha256:8f5c07333d543103541ba7be0e2ce16eeee8130cb0b3f9238ab904ce1e85baff", size = 805984 } -wheels = [ - { url = "https://files.pythonhosted.org/packages/89/aa/ab0f7891a01eeb2d2e338ae8fecbe57fcebea1a24dbb64d45801bfab481d/attrs-24.3.0-py3-none-any.whl", hash = "sha256:ac96cd038792094f438ad1f6ff80837353805ac950cd2aa0e0625ef19850c308", size = 63397 }, -] - [[package]] name = "bcrypt" version = "4.2.1" @@ -203,9 +194,7 @@ name = "filesystem-client" version = "0.0" source = { virtual = "." } dependencies = [ - { name = "jsonschema" }, { name = "ops" }, - { name = "rpds-py" }, ] [package.optional-dependencies] @@ -225,7 +214,6 @@ dev = [ requires-dist = [ { name = "codespell", marker = "extra == 'dev'" }, { name = "coverage", extras = ["toml"], marker = "extra == 'dev'" }, - { name = "jsonschema", specifier = "~=4.23.0" }, { name = "juju", marker = "extra == 'dev'" }, { name = "ops", specifier = "~=2.8" }, { name = "ops", extras = ["testing"], marker = "extra == 'dev'" }, @@ -233,7 +221,6 @@ requires-dist = [ { name = "pytest", marker = "extra == 'dev'" }, { name = "pytest-operator", marker = "extra == 'dev'" }, { name = "pytest-order", marker = "extra == 'dev'" }, - { name = "rpds-py", specifier = "~=0.22.3" }, { name = "ruff", marker = "extra == 'dev'" }, ] @@ -338,33 +325,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/31/80/3a54838c3fb461f6fec263ebf3a3a41771bd05190238de3486aae8540c36/jinja2-3.1.4-py3-none-any.whl", hash = "sha256:bc5dd2abb727a5319567b7a813e6a2e7318c39f4f487cfe6c89c6f9c7d25197d", size = 133271 }, ] -[[package]] -name = "jsonschema" -version = "4.23.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "attrs" }, - { name = "jsonschema-specifications" }, - { name = "referencing" }, - { name = "rpds-py" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/38/2e/03362ee4034a4c917f697890ccd4aec0800ccf9ded7f511971c75451deec/jsonschema-4.23.0.tar.gz", hash = "sha256:d71497fef26351a33265337fa77ffeb82423f3ea21283cd9467bb03999266bc4", size = 325778 } -wheels = [ - { url = "https://files.pythonhosted.org/packages/69/4a/4f9dbeb84e8850557c02365a0eee0649abe5eb1d84af92a25731c6c0f922/jsonschema-4.23.0-py3-none-any.whl", hash = "sha256:fbadb6f8b144a8f8cf9f0b89ba94501d143e50411a1278633f56a7acf7fd5566", size = 88462 }, -] - -[[package]] -name = "jsonschema-specifications" -version = "2024.10.1" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "referencing" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/10/db/58f950c996c793472e336ff3655b13fbcf1e3b359dcf52dcf3ed3b52c352/jsonschema_specifications-2024.10.1.tar.gz", hash = "sha256:0f38b83639958ce1152d02a7f062902c41c8fd20d558b0c34344292d417ae272", size = 15561 } -wheels = [ - { url = "https://files.pythonhosted.org/packages/d1/0f/8910b19ac0670a0f80ce1008e5e751c4a57e14d2c4c13a482aa6079fa9d6/jsonschema_specifications-2024.10.1-py3-none-any.whl", hash = "sha256:a09a0680616357d9a0ecf05c12ad234479f549239d0f5b55f3deea67475da9bf", size = 18459 }, -] - [[package]] name = "juju" version = "3.6.0.0" @@ -581,16 +541,16 @@ wheels = [ [[package]] name = "protobuf" -version = "5.29.1" +version = "5.29.2" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/d2/4f/1639b7b1633d8fd55f216ba01e21bf2c43384ab25ef3ddb35d85a52033e8/protobuf-5.29.1.tar.gz", hash = "sha256:683be02ca21a6ffe80db6dd02c0b5b2892322c59ca57fd6c872d652cb80549cb", size = 424965 } +sdist = { url = "https://files.pythonhosted.org/packages/a5/73/4e6295c1420a9d20c9c351db3a36109b4c9aa601916cb7c6871e3196a1ca/protobuf-5.29.2.tar.gz", hash = "sha256:b2cc8e8bb7c9326996f0e160137b0861f1a82162502658df2951209d0cb0309e", size = 424901 } wheels = [ - { url = "https://files.pythonhosted.org/packages/50/c7/28669b04691a376cf7d0617d612f126aa0fff763d57df0142f9bf474c5b8/protobuf-5.29.1-cp310-abi3-win32.whl", hash = "sha256:22c1f539024241ee545cbcb00ee160ad1877975690b16656ff87dde107b5f110", size = 422706 }, - { url = "https://files.pythonhosted.org/packages/e3/33/dc7a7712f457456b7e0b16420ab8ba1cc8686751d3f28392eb43d0029ab9/protobuf-5.29.1-cp310-abi3-win_amd64.whl", hash = "sha256:1fc55267f086dd4050d18ef839d7bd69300d0d08c2a53ca7df3920cc271a3c34", size = 434505 }, - { url = "https://files.pythonhosted.org/packages/e5/39/44239fb1c6ec557e1731d996a5de89a9eb1ada7a92491fcf9c5d714052ed/protobuf-5.29.1-cp38-abi3-macosx_10_9_universal2.whl", hash = "sha256:d473655e29c0c4bbf8b69e9a8fb54645bc289dead6d753b952e7aa660254ae18", size = 417822 }, - { url = "https://files.pythonhosted.org/packages/fb/4a/ec56f101d38d4bef2959a9750209809242d86cf8b897db00f2f98bfa360e/protobuf-5.29.1-cp38-abi3-manylinux2014_aarch64.whl", hash = "sha256:b5ba1d0e4c8a40ae0496d0e2ecfdbb82e1776928a205106d14ad6985a09ec155", size = 319572 }, - { url = "https://files.pythonhosted.org/packages/04/52/c97c58a33b3d6c89a8138788576d372a90a6556f354799971c6b4d16d871/protobuf-5.29.1-cp38-abi3-manylinux2014_x86_64.whl", hash = "sha256:8ee1461b3af56145aca2800e6a3e2f928108c749ba8feccc6f5dd0062c410c0d", size = 319671 }, - { url = "https://files.pythonhosted.org/packages/3b/24/c8c49df8f6587719e1d400109b16c10c6902d0c9adddc8fff82840146f99/protobuf-5.29.1-py3-none-any.whl", hash = "sha256:32600ddb9c2a53dedc25b8581ea0f1fd8ea04956373c0c07577ce58d312522e0", size = 172547 }, + { url = "https://files.pythonhosted.org/packages/f3/42/6db5387124708d619ffb990a846fb123bee546f52868039f8fa964c5bc54/protobuf-5.29.2-cp310-abi3-win32.whl", hash = "sha256:c12ba8249f5624300cf51c3d0bfe5be71a60c63e4dcf51ffe9a68771d958c851", size = 422697 }, + { url = "https://files.pythonhosted.org/packages/6c/38/2fcc968b377b531882d6ab2ac99b10ca6d00108394f6ff57c2395fb7baff/protobuf-5.29.2-cp310-abi3-win_amd64.whl", hash = "sha256:842de6d9241134a973aab719ab42b008a18a90f9f07f06ba480df268f86432f9", size = 434495 }, + { url = "https://files.pythonhosted.org/packages/cb/26/41debe0f6615fcb7e97672057524687ed86fcd85e3da3f031c30af8f0c51/protobuf-5.29.2-cp38-abi3-macosx_10_9_universal2.whl", hash = "sha256:a0c53d78383c851bfa97eb42e3703aefdc96d2036a41482ffd55dc5f529466eb", size = 417812 }, + { url = "https://files.pythonhosted.org/packages/e4/20/38fc33b60dcfb380507b99494aebe8c34b68b8ac7d32808c4cebda3f6f6b/protobuf-5.29.2-cp38-abi3-manylinux2014_aarch64.whl", hash = "sha256:494229ecd8c9009dd71eda5fd57528395d1eacdf307dbece6c12ad0dd09e912e", size = 319562 }, + { url = "https://files.pythonhosted.org/packages/90/4d/c3d61e698e0e41d926dbff6aa4e57428ab1a6fc3b5e1deaa6c9ec0fd45cf/protobuf-5.29.2-cp38-abi3-manylinux2014_x86_64.whl", hash = "sha256:b6b0d416bbbb9d4fbf9d0561dbfc4e324fd522f61f7af0fe0f282ab67b22477e", size = 319662 }, + { url = "https://files.pythonhosted.org/packages/f3/fd/c7924b4c2a1c61b8f4b64edd7a31ffacf63432135a2606f03a2f0d75a750/protobuf-5.29.2-py3-none-any.whl", hash = "sha256:fde4554c0e578a5a0bcc9a276339594848d1e89f9ea47b4427c80e5d72f90181", size = 172539 }, ] [[package]] @@ -697,15 +657,15 @@ wheels = [ [[package]] name = "pyright" -version = "1.1.390" +version = "1.1.391" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "nodeenv" }, { name = "typing-extensions" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/ba/42/1e0392f35dd275f9f775baf7c86407cef7f6a0d9b8e099a93e5422a7e571/pyright-1.1.390.tar.gz", hash = "sha256:aad7f160c49e0fbf8209507a15e17b781f63a86a1facb69ca877c71ef2e9538d", size = 21950 } +sdist = { url = "https://files.pythonhosted.org/packages/11/05/4ea52a8a45cc28897edb485b4102d37cbfd5fce8445d679cdeb62bfad221/pyright-1.1.391.tar.gz", hash = "sha256:66b2d42cdf5c3cbab05f2f4b76e8bec8aa78e679bfa0b6ad7b923d9e027cadb2", size = 21965 } wheels = [ - { url = "https://files.pythonhosted.org/packages/43/20/3f492ca789fb17962ad23619959c7fa642082621751514296c58de3bb801/pyright-1.1.390-py3-none-any.whl", hash = "sha256:ecebfba5b6b50af7c1a44c2ba144ba2ab542c227eb49bc1f16984ff714e0e110", size = 18579 }, + { url = "https://files.pythonhosted.org/packages/ad/89/66f49552fbeb21944c8077d11834b2201514a56fd1b7747ffff9630f1bd9/pyright-1.1.391-py3-none-any.whl", hash = "sha256:54fa186f8b3e8a55a44ebfa842636635688670c6896dcf6cf4a7fc75062f4d15", size = 18579 }, ] [[package]] @@ -802,19 +762,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/0c/e8/4f648c598b17c3d06e8753d7d13d57542b30d56e6c2dedf9c331ae56312e/PyYAML-6.0.2-cp312-cp312-win_amd64.whl", hash = "sha256:7e7401d0de89a9a855c839bc697c079a4af81cf878373abd7dc625847d25cbd8", size = 156338 }, ] -[[package]] -name = "referencing" -version = "0.35.1" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "attrs" }, - { name = "rpds-py" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/99/5b/73ca1f8e72fff6fa52119dbd185f73a907b1989428917b24cff660129b6d/referencing-0.35.1.tar.gz", hash = "sha256:25b42124a6c8b632a425174f24087783efb348a6f1e0008e63cd4466fedf703c", size = 62991 } -wheels = [ - { url = "https://files.pythonhosted.org/packages/b7/59/2056f61236782a2c86b33906c025d4f4a0b17be0161b63b70fd9e8775d36/referencing-0.35.1-py3-none-any.whl", hash = "sha256:eda6d3234d62814d1c64e305c1331c9a3a6132da475ab6382eaa997b21ee75de", size = 26684 }, -] - [[package]] name = "requests" version = "2.32.3" @@ -843,27 +790,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/3b/5d/63d4ae3b9daea098d5d6f5da83984853c1bbacd5dc826764b249fe119d24/requests_oauthlib-2.0.0-py2.py3-none-any.whl", hash = "sha256:7dd8a5c40426b779b0868c404bdef9768deccf22749cde15852df527e6269b36", size = 24179 }, ] -[[package]] -name = "rpds-py" -version = "0.22.3" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/01/80/cce854d0921ff2f0a9fa831ba3ad3c65cee3a46711addf39a2af52df2cfd/rpds_py-0.22.3.tar.gz", hash = "sha256:e32fee8ab45d3c2db6da19a5323bc3362237c8b653c70194414b892fd06a080d", size = 26771 } -wheels = [ - { url = "https://files.pythonhosted.org/packages/75/47/3383ee3bd787a2a5e65a9b9edc37ccf8505c0a00170e3a5e6ea5fbcd97f7/rpds_py-0.22.3-cp312-cp312-macosx_10_12_x86_64.whl", hash = "sha256:27e98004595899949bd7a7b34e91fa7c44d7a97c40fcaf1d874168bb652ec67e", size = 352334 }, - { url = "https://files.pythonhosted.org/packages/40/14/aa6400fa8158b90a5a250a77f2077c0d0cd8a76fce31d9f2b289f04c6dec/rpds_py-0.22.3-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:1978d0021e943aae58b9b0b196fb4895a25cc53d3956b8e35e0b7682eefb6d56", size = 342111 }, - { url = "https://files.pythonhosted.org/packages/7d/06/395a13bfaa8a28b302fb433fb285a67ce0ea2004959a027aea8f9c52bad4/rpds_py-0.22.3-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:655ca44a831ecb238d124e0402d98f6212ac527a0ba6c55ca26f616604e60a45", size = 384286 }, - { url = "https://files.pythonhosted.org/packages/43/52/d8eeaffab047e6b7b7ef7f00d5ead074a07973968ffa2d5820fa131d7852/rpds_py-0.22.3-cp312-cp312-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:feea821ee2a9273771bae61194004ee2fc33f8ec7db08117ef9147d4bbcbca8e", size = 391739 }, - { url = "https://files.pythonhosted.org/packages/83/31/52dc4bde85c60b63719610ed6f6d61877effdb5113a72007679b786377b8/rpds_py-0.22.3-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:22bebe05a9ffc70ebfa127efbc429bc26ec9e9b4ee4d15a740033efda515cf3d", size = 427306 }, - { url = "https://files.pythonhosted.org/packages/70/d5/1bab8e389c2261dba1764e9e793ed6830a63f830fdbec581a242c7c46bda/rpds_py-0.22.3-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:3af6e48651c4e0d2d166dc1b033b7042ea3f871504b6805ba5f4fe31581d8d38", size = 442717 }, - { url = "https://files.pythonhosted.org/packages/82/a1/a45f3e30835b553379b3a56ea6c4eb622cf11e72008229af840e4596a8ea/rpds_py-0.22.3-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e67ba3c290821343c192f7eae1d8fd5999ca2dc99994114643e2f2d3e6138b15", size = 385721 }, - { url = "https://files.pythonhosted.org/packages/a6/27/780c942de3120bdd4d0e69583f9c96e179dfff082f6ecbb46b8d6488841f/rpds_py-0.22.3-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:02fbb9c288ae08bcb34fb41d516d5eeb0455ac35b5512d03181d755d80810059", size = 415824 }, - { url = "https://files.pythonhosted.org/packages/94/0b/aa0542ca88ad20ea719b06520f925bae348ea5c1fdf201b7e7202d20871d/rpds_py-0.22.3-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:f56a6b404f74ab372da986d240e2e002769a7d7102cc73eb238a4f72eec5284e", size = 561227 }, - { url = "https://files.pythonhosted.org/packages/0d/92/3ed77d215f82c8f844d7f98929d56cc321bb0bcfaf8f166559b8ec56e5f1/rpds_py-0.22.3-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:0a0461200769ab3b9ab7e513f6013b7a97fdeee41c29b9db343f3c5a8e2b9e61", size = 587424 }, - { url = "https://files.pythonhosted.org/packages/09/42/cacaeb047a22cab6241f107644f230e2935d4efecf6488859a7dd82fc47d/rpds_py-0.22.3-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:8633e471c6207a039eff6aa116e35f69f3156b3989ea3e2d755f7bc41754a4a7", size = 555953 }, - { url = "https://files.pythonhosted.org/packages/e6/52/c921dc6d5f5d45b212a456c1f5b17df1a471127e8037eb0972379e39dff4/rpds_py-0.22.3-cp312-cp312-win32.whl", hash = "sha256:593eba61ba0c3baae5bc9be2f5232430453fb4432048de28399ca7376de9c627", size = 221339 }, - { url = "https://files.pythonhosted.org/packages/f2/c7/f82b5be1e8456600395366f86104d1bd8d0faed3802ad511ef6d60c30d98/rpds_py-0.22.3-cp312-cp312-win_amd64.whl", hash = "sha256:d115bffdd417c6d806ea9069237a4ae02f513b778e3789a359bc5856e0404cc4", size = 235786 }, -] - [[package]] name = "rsa" version = "4.9"