From a44035a3fa610f4ac6c9287c005fd7a7c08970b0 Mon Sep 17 00:00:00 2001 From: Teodora Mihoc Date: Fri, 29 Nov 2024 13:11:13 +0100 Subject: [PATCH 01/16] docs: add and reformat discourse docs --- docs/explanation/holistic-vs-delta-charms.md | 66 ++ .../how-and-when-to-defer-events.md | 48 + docs/explanation/index.md | 11 + .../storedstate-uses-limitations.md | 104 ++ docs/howto/get-started-with-charm-testing.md | 332 +++++++ docs/howto/index.md | 29 + ...ument-your-charm-with-tracing-telemetry.md | 146 +++ docs/howto/manage-actions.md | 215 ++++ docs/howto/manage-configurations.md | 122 +++ docs/howto/manage-interfaces.md | 482 +++++++++ docs/howto/manage-leadership-changes.md | 149 +++ docs/howto/manage-libraries.md | 225 +++++ docs/howto/manage-logs.md | 66 ++ docs/howto/manage-relations.md | 343 +++++++ docs/howto/manage-resources.md | 70 ++ docs/howto/manage-secrets.md | 377 +++++++ docs/howto/manage-storage.md | 247 +++++ docs/howto/manage-the-charm-version.md | 87 ++ docs/howto/manage-the-workload-version.md | 154 +++ .../run-workloads-with-a-charm-kubernetes.md | 938 ++++++++++++++++++ .../run-workloads-with-a-charm-machines.md | 81 ++ ...n-a-hooks-based-charm-into-an-ops-charm.md | 302 ++++++ .../write-integration-tests-for-a-charm.md | 399 ++++++++ .../howto/write-scenario-tests-for-a-charm.md | 81 ++ docs/howto/write-unit-tests-for-a-charm.md | 195 ++++ docs/index.md | 92 ++ docs/{index.rst => ops.rst} | 0 docs/pebble.rst | 6 - docs/reference/api/index.md | 13 + docs/reference/api/ops-main-entrypoint.rst | 16 + .../api/ops-testing-harness.rst} | 4 +- .../api/ops-testing.rst} | 4 +- docs/reference/api/ops.rst | 8 + docs/reference/api/pebble.rst | 6 + docs/reference/charm-relation-interfaces.md | 44 + docs/reference/index.md | 14 + docs/reference/interface-tests.md | 39 + docs/reference/pytest-operator.md | 19 + docs/reference/testing.md | 90 ++ .../create-a-minimal-kubernetes-charm.md | 435 ++++++++ .../expose-operational-tasks-via-actions.md | 166 ++++ ...on-of-the-application-behind-your-charm.md | 118 +++ .../index.md | 89 ++ .../integrate-your-charm-with-postgresql.md | 389 ++++++++ .../make-your-charm-configurable.md | 211 ++++ .../observe-your-charm-with-cos-lite.md | 470 +++++++++ .../open-a-kubernetes-port-in-your-charm.md | 359 +++++++ .../preserve-your-charms-data.md | 170 ++++ .../publish-your-charm-on-charmhub.md | 192 ++++ .../set-up-your-development-environment.md | 30 + .../study-your-application.md | 105 ++ .../write-integration-tests-for-your-charm.md | 204 ++++ .../write-scenario-tests-for-your-charm.md | 204 ++++ .../write-unit-tests-for-your-charm.md | 200 ++++ docs/tutorial/index.md | 12 + .../write-your-first-machine-charm.md | 530 ++++++++++ 56 files changed, 9498 insertions(+), 10 deletions(-) create mode 100644 docs/explanation/holistic-vs-delta-charms.md create mode 100644 docs/explanation/how-and-when-to-defer-events.md create mode 100644 docs/explanation/index.md create mode 100644 docs/explanation/storedstate-uses-limitations.md create mode 100644 docs/howto/get-started-with-charm-testing.md create mode 100644 docs/howto/index.md create mode 100644 docs/howto/instrument-your-charm-with-tracing-telemetry.md create mode 100644 docs/howto/manage-actions.md create mode 100644 docs/howto/manage-configurations.md create mode 100644 docs/howto/manage-interfaces.md create mode 100644 docs/howto/manage-leadership-changes.md create mode 100644 docs/howto/manage-libraries.md create mode 100644 docs/howto/manage-logs.md create mode 100644 docs/howto/manage-relations.md create mode 100644 docs/howto/manage-resources.md create mode 100644 docs/howto/manage-secrets.md create mode 100644 docs/howto/manage-storage.md create mode 100644 docs/howto/manage-the-charm-version.md create mode 100644 docs/howto/manage-the-workload-version.md create mode 100644 docs/howto/run-workloads-with-a-charm-kubernetes.md create mode 100644 docs/howto/run-workloads-with-a-charm-machines.md create mode 100644 docs/howto/turn-a-hooks-based-charm-into-an-ops-charm.md create mode 100644 docs/howto/write-integration-tests-for-a-charm.md create mode 100644 docs/howto/write-scenario-tests-for-a-charm.md create mode 100644 docs/howto/write-unit-tests-for-a-charm.md create mode 100644 docs/index.md rename docs/{index.rst => ops.rst} (100%) delete mode 100644 docs/pebble.rst create mode 100644 docs/reference/api/index.md create mode 100644 docs/reference/api/ops-main-entrypoint.rst rename docs/{harness.rst => reference/api/ops-testing-harness.rst} (95%) rename docs/{state-transition-testing.rst => reference/api/ops-testing.rst} (98%) create mode 100644 docs/reference/api/ops.rst create mode 100644 docs/reference/api/pebble.rst create mode 100644 docs/reference/charm-relation-interfaces.md create mode 100644 docs/reference/index.md create mode 100644 docs/reference/interface-tests.md create mode 100644 docs/reference/pytest-operator.md create mode 100644 docs/reference/testing.md create mode 100644 docs/tutorial/from-zero-to-hero-write-your-first-kubernetes-charm/create-a-minimal-kubernetes-charm.md create mode 100644 docs/tutorial/from-zero-to-hero-write-your-first-kubernetes-charm/expose-operational-tasks-via-actions.md create mode 100644 docs/tutorial/from-zero-to-hero-write-your-first-kubernetes-charm/expose-the-version-of-the-application-behind-your-charm.md create mode 100644 docs/tutorial/from-zero-to-hero-write-your-first-kubernetes-charm/index.md create mode 100644 docs/tutorial/from-zero-to-hero-write-your-first-kubernetes-charm/integrate-your-charm-with-postgresql.md create mode 100644 docs/tutorial/from-zero-to-hero-write-your-first-kubernetes-charm/make-your-charm-configurable.md create mode 100644 docs/tutorial/from-zero-to-hero-write-your-first-kubernetes-charm/observe-your-charm-with-cos-lite.md create mode 100644 docs/tutorial/from-zero-to-hero-write-your-first-kubernetes-charm/open-a-kubernetes-port-in-your-charm.md create mode 100644 docs/tutorial/from-zero-to-hero-write-your-first-kubernetes-charm/preserve-your-charms-data.md create mode 100644 docs/tutorial/from-zero-to-hero-write-your-first-kubernetes-charm/publish-your-charm-on-charmhub.md create mode 100644 docs/tutorial/from-zero-to-hero-write-your-first-kubernetes-charm/set-up-your-development-environment.md create mode 100644 docs/tutorial/from-zero-to-hero-write-your-first-kubernetes-charm/study-your-application.md create mode 100644 docs/tutorial/from-zero-to-hero-write-your-first-kubernetes-charm/write-integration-tests-for-your-charm.md create mode 100644 docs/tutorial/from-zero-to-hero-write-your-first-kubernetes-charm/write-scenario-tests-for-your-charm.md create mode 100644 docs/tutorial/from-zero-to-hero-write-your-first-kubernetes-charm/write-unit-tests-for-your-charm.md create mode 100644 docs/tutorial/index.md create mode 100644 docs/tutorial/write-your-first-machine-charm.md diff --git a/docs/explanation/holistic-vs-delta-charms.md b/docs/explanation/holistic-vs-delta-charms.md new file mode 100644 index 000000000..e7444d646 --- /dev/null +++ b/docs/explanation/holistic-vs-delta-charms.md @@ -0,0 +1,66 @@ +(holistic-vs-delta-charms)= +# Holistic vs delta charms + + +Charm developers have had many discussion about "holistic" charms compared to "delta" charms, and which approach is better. First, let's define those terms: + +* A *delta-based* charm is when the charm handles each kind of Juju hook with a separate handler function, which does the minimum necessary to process that kind of event. +* A *holistic* charm handles some or all Juju hooks using a common code path such as `_update_charm`, which queries the charm config and relation data and "rewrites the world", that is, rewrites application configuration and restarts necessary services. + +Juju itself nudges charm authors in the direction of delta-based charms, because it provides specific event kinds that signal that one "thing" changed: `config-changed` says that a config value changed, `relation-changed` says that relation data has changed, `pebble-ready` signals that the Pebble container is ready, and so on. + +However, this only goes so far: `config-changed` doesn't tell the charm which config keys changed, and `relation-changed` doesn't tell the charm how the relation data changed. + +In addition, the charm may receive an event like `config-changed` before it's ready to handle it, for example, if the container is not yet ready (`pebble-ready` has not yet been triggered). In such cases, charms could try to wait for both events to occur, possibly storing state to track which events have occurred -- but that is error-prone. + +Alternatively, a charm can use a holistic approach and handle both `config-changed` and `pebble-ready` with a single code path, as in this example: + +```python +class MyCharm(ops.CharmBase): + def __init__(self, framework: ops.Framework): + super().__init__(framework) + framework.observe(self.on.config_changed, self._update_charm) + framework.observe(self.on['redis'].pebble_ready, self._update_charm) + + def _update_charm(self, _: ops.EventBase): # event parameter isn't used + redis_port = self.config.get('redis-port') + if not redis_port: + # pebble-ready happened first, wait for config-changed + return + + # If both the Pebble container and config are ready, rewrite the + # container's config file and restart Redis if needed. + container = self.unit.get_container('redis') + try: + self._update_redis_config(container, redis_port) + except ops.pebble.ConnectionError: + # config-changed happened first, wait for pebble-ready + return +``` + + +## When to use the holistic approach + +If a charm is waiting for a collection of events, as in the example above, it makes sense to group those events together and handle them holistically, with a single code path. + +In other words, when writing a charm, it's not so much "should the *charm* be holistic?" as "does it make sense for *these events* to be handled holistically?" + +Using the holistic approach is normally centred around configuring an application. Various events that affect configuration use a common handler, to simplify writing an application config file and restarting the application. This is common for events like `config-changed`, `relation-changed`, `secret-changed`, and `pebble-ready`. + +Many existing charms use holistic event handling. A few examples are: + +- [`alertmanager-k8s` uses a `_common_exit_hook` method to unify several event handlers](https://github.com/canonical/alertmanager-k8s-operator/blob/561f1d8eb1dc6e4511c1c0b3cba444a3ec399464/src/charm.py#L390) +- [`hello-kubecon` is a simple charm that handles `config-changed` and `pebble-cready` holistically](https://github.com/jnsgruk/hello-kubecon/blob/dbd133466dde59ee64f20a732a8f3d2e560ec3b8/src/charm.py#L32-L33) +- [`prometheus-k8s` uses a common `_configure` method to handle various events](https://github.com/canonical/prometheus-k8s-operator/blob/84c6a406ed585cdb7ba40e01a258864987d6f67f/src/charm.py#L221-L230) +- [`sdcore-gnbsim-k8s` also uses a common `_configure` method](https://github.com/canonical/sdcore-gnbsim-k8s-operator/blob/ea2afe069346757b1eb6c02de5b4f50f90e81698/src/charm.py#L84-L92) + + +## Which events can be handled holistically? + +Only some events make sense to handle holistically. For example, `remove` is triggered when a unit is about to be terminated, so it doesn't make sense to handle it holistically. + +Similarly, events like `secret-expired` and `secret-rotate` don't make sense to handle holistically, because the charm must do something specific in response to the event. For example, Juju will keep triggering `secret-expired` until the charm creates a new secret revision by calling [`event.secret.set_content()`](https://ops.readthedocs.io/en/latest/#ops.Secret.set_content). + +This is very closely related to [which events can be `defer`red](https://juju.is/docs/sdk/how-and-when-to-defer-events). A good rule of thumb is this: if an event can be deferred, it may make sense to handle it holistically. + +On the other hand, if an event cannot be deferred, the charm cannot handle it holistically. This applies to action "events", `stop`, `remove`, `secret-expired`, `secret-rotate`, and Ops-emitted events such as `collect-status`. \ No newline at end of file diff --git a/docs/explanation/how-and-when-to-defer-events.md b/docs/explanation/how-and-when-to-defer-events.md new file mode 100644 index 000000000..87c8f1e6e --- /dev/null +++ b/docs/explanation/how-and-when-to-defer-events.md @@ -0,0 +1,48 @@ +(how-and-when-to-defer-events)= +# How, and when, to defer events + +Deferring an event is a common pattern, and when used appropriately is a convenient tool for charmers. However, there are limitations to `defer()` - in particular, that the charm has no way to specify when the handler will be re-run, and that event ordering and context move away from the expected pattern. Our advice is that `defer()` is a good solution for some problems, but is best avoided for others. + +## Good: retrying on temporary failure + +If the charm encounters a temporary failure (such as working with a container or an external API), and expects that the failure may be very short lived, our recommendation is to retry several times for up to a second. If the failure continues, but the charm still expects that it will be resolved without any intervention from a human, then deferring the event handler is often a good choice - along with placing the unit or app in waiting status. + +Note that it’s important to consider that when the deferred handler is run again, the Juju context may not be exactly the same as it was when the event was first emitted, so the charm code needs to be aware of this. + +If the temporary failure is because the workload is busy, and the charm is deployed to a Kubernetes sidecar controller, you might be able to avoid the defer using a [Pebble custom notice](https://juju.is/docs/sdk/interact-with-pebble#heading--use-custom-notices-from-the-workload-container). For example, if the code can’t continue because the workload is currently restarting, if you can have a post-completion hook for the restart that executes `pebble notify`, then you can ensure that the charm is ‘woken up’ at the right time to handle the work. + +In the future, we hope to see a Juju ‘request re-emit event’ feature that will let the charm tell Juju when it expects the problem to be resolved. + +## Reconsider: sequencing + +There are some situations where sequencing of units needs to be arranged - for example, to restart replicas before a primary is restarted. Deferring a handler can be used to manage this situation. However, sequencing can also be arranged using a peer relation, and there’s a convenient [rolling-ops charm lib](https://github.com/canonical/charm-rolling-ops) that implements this for you, and we recommend using that approach first. + +Using a peer relation to orchestrate the rolling operation allows for more fine-grained control than a simple defer, and avoids the issue of not having control over when the deferred handler will be re-run. + +## Reconsider: waiting for a collection of events + +It’s common for charms to need a collection of information in order to configure the application (for example, to write a configuration file). For example, the configuration might require a user-set config value, a secret provided by a relation, and a Kubernetes sidecar container to be ready. + +Rather than having the handlers for each of these events (`config-changed`, `secret-changed` and/or `relation-changed`, `pebble-ready`) defer if other parts of the configuration are not yet available, it’s best to have the charm observe all three events and set the unit or app state to waiting, maintenance, or blocked status (or have the `collect-status` handler do this) and return. When the last piece of information is available, the handler that notifies the charm of that will complete the work. This is commonly called the "holistic" event handling pattern. + +Avoiding defer means that there isn’t a queue of deferred handlers that all do the same work - for example, if `config-changed`, `relation-changed`, and `pebble-ready` were all deferred then when they were all ready, they would all run successfully. This is particularly important when the work is expensive - such as an application restart after writing the configuration, so should not be done unnecessarily. + +## OK: waiting without expecting a follow-up event + +In some situations, the charm is waiting for a system to be ready, but it’s not one that will trigger a Juju event (as in the case above). For example, the charm might need the workload application to be fully started up, and that might happen after all of the initial start, `config-changed`, `relation-joined`, `pebble-ready`, etc events. + +Deferring the work here is ok, but it’s important to consider the delay between deferring the event and its eventual re-emitting - it’s not safe to assume that this will be a small period of time, unless you know that another event can be expected. + +For a Kubernetes charm, If the charm is waiting on the workload and it’s possible to have the workload execute a command when it’s ready, then using a [Pebble custom notice](https://juju.is/docs/sdk/interact-with-pebble#heading--use-custom-notices-from-the-workload-container) is much better than deferring. This then becomes another example of “waiting for a collection of events”, described above. + +## Not possible: actions, shutting down, framework generated events, secrets + +In some situations, it’s not possible to defer an event, and attempting to do so will raise a `RuntimeError`. + +In some cases, this is because the events are run with every Juju hook event, such as `pre-commit`, `commit`, and `update-status`. In others, it’s because Juju provides a built-in retry mechanism, such as `secret-expired` and `secret-rotate`. + +With actions, there’s an expectation that the action either succeeds or fails immediately, and there are mechanisms for communicating directly with the user that initiated the action (`event.log` and `event.set_results`). This means that deferring an action event doesn’t make sense. + +Finally, when doing cleanup during the shutdown phase of a charm’s lifecycle, deferring isn’t practical with the current implementation, where it’s tied to future events. For `remove`, for example, the unit will no longer exist after the event, so there will not be any future events that can trigger the deferred one - if there’s work that has to be done before the unit is gone, then you’ll need to enter an error state instead. The stop event is followed by remove, and possibly a few other events, but likewise has few chances to be re-emitted. + +Note that all deferred events vanish when the unit is removed, so the charm code needs to take this into consideration. diff --git a/docs/explanation/index.md b/docs/explanation/index.md new file mode 100644 index 000000000..0238e87d8 --- /dev/null +++ b/docs/explanation/index.md @@ -0,0 +1,11 @@ +(explanation)= +# Explanation + +```{toctree} +:maxdepth: 1 + +holistic-vs-delta-charms +how-and-when-to-defer-events +storedstate-uses-limitations +``` + diff --git a/docs/explanation/storedstate-uses-limitations.md b/docs/explanation/storedstate-uses-limitations.md new file mode 100644 index 000000000..a81ca4093 --- /dev/null +++ b/docs/explanation/storedstate-uses-limitations.md @@ -0,0 +1,104 @@ +(storedstate-uses-limitations)= +# StoredState: Uses, Limitations + +... and why charm authors should avoid state when they can. + +## Purpose of this doc + +This is an explanatory doc covering how charm authors might track local state in a Juju unit. We'll cover the Operator Framework's concept of [`StoredState`](https://juju.is/docs/sdk/constructs#heading--stored-state), along with some differences in how it works between machine charms and Kubernetes charms. We'll talk about [Peer Relations](https://juju.is/docs/sdk/relations#heading--peer-relations) as an alternative for storing some kinds of information, and also talk about how charm authors probably should avoid recording state when they can avoid doing so. Relying on the SDK's built in caching facilities is generally the preferred direction for a charm. + +## A trivial example + +We'll begin by setting up a simple scenario. A charm author would like to charm up a (made up) service called `ExampleBlog`. The ideal cloud service is stateless and immutable, but `ExampleBlog` has some state: it can run in either a `production` mode or a `test` mode. + +The standard way to set ExampleBlog's mode is to write either the string `test` or `production` to `/etc/example_blog/mode`, then restart the service. Leaving aside whether this is *advisable* behavior, this is how `ExampleBlog` works, and an `ExampleBlog` veteran user would expect a `ExampleBlog` charm to allow them to toggle modes by writing to that config file. (I sense a patch to upstream brewing, but let's assume, for our example, that we can't dynamically load the config.) + +Here's a simplified charm code snippet that will allow us to toggle the state of an already running instance of `ExampleBlog`. + +```python +def _on_config_changed(self, event): + mode = self.model.config['mode'] + + with open('/etc/example_blog/mode', 'w') as mode_file: + mode_file.write(f'{mode}\n') + + self._restart() +``` + +Assume that `_restart` does something sensible to restart the service -- e.g., calls `service_restart` from the [systemd](https://charmhub.io/operator-libs-linux/libraries/systemd) library in a machine version of this charm. + +## A problematic solution + +The problem with the code as written is that the `ExampleBlog` daemon will restart every time the config-changed hooked fires. That's definitely unwanted downtime! We might be tempted to solve the issue with `StoredState`: + +```python +def __init__(self, *args): + super().__init__(*args) + self._stored.set_default(current_mode="test") + +def _on_config_changed(self, event): + mode = self.model.config['mode'] + if self._stored.current_mode == mode: + return + + with open('/etc/example_blog/mode', 'w') as mode_file: + mode_file.write('{}\n'.format(mode) + + self._restart() + + self._stored.current_mode = mode +``` + +The `StoredState` [docs](https://juju.is/docs/sdk/constructs#heading--stored-state) advise against doing this, for good reason. We have added one to the list of places that attempt to track `ExampleBlog`'s "mode". In addition to the config file on disk, the juju config, and the actual state of the running code, we've added a fourth "instance" of the state: "current_mode" in our `StoredState` object. We've doubled the number of possible states of this part of the system from 8 to 16, without increasing the number of correct states. There are still only two: all set to `test`, or all set to `production`. We have essentially halved the reliability of this part of our code. + +## Differences in StoredState behaviour across substrates + +Let's say the charm is running on Kubernetes, and the container it is running in gets destroyed and recreated. This might happen due to events outside of an operator's control -- perhaps the underlying Kubernetes service rescheduled the pod, for example. In this scenario the `StoredState` will go away, and the flags will be reset. + +Do you see the bug in our example code? We could fix it by setting the initial value in our `StoredState` to something other than `test` or `production`. E.g., `self._stored.set_default(current_mode="unset")`. This will never match the actual intended state, and we'll thus always invoke the codepath that loads the operator's intended state after a pod restart, and write that to the new local disk. + +What if we are tracking some piece of information that *should* survive a pod restart? + +In this case, charm authors can pass `use_juju_for_storage=True` to the charm's `main` routine ([example](https://github.com/canonical/alertmanager-k8s-operator/blob/8371a1424c0a73d62d249ca085edf693c8084279/src/charm.py#L454)). This will allocate some space on the controller to store per unit data, and that data will persist through events that could kill and recreate the underlying pod. Keep in mind that this can cause trouble! In the case of `ExampleBlog`, we clearly would not want the `StoredState` record for "mode" to survive a pod restart -- the correct state is already appropriately stored in Juju's config, and stale state in the controller's storage might result in the charm skipping a necessary config write and restart cycle. + +## Practical suggestions and solutions + +_Most of the time, charm authors should not track state in a charm._ + +More specifically, authors should only use `StoredState` when they are certain that the charm can handle any cache consistency issues, and that tracking the state is actually saving a significant number of unneeded CPU cycles. + +In our example code, for instance, we might think about the fact that `config_changed` hooks, even in a busy cloud, fire with a frequency measured in seconds. It's not particularly expensive to read the contents of a small file every few seconds, and so we might implement the following, which is stateless (or at least, does not hold state in the charm): + +```python +def _on_config_changed(self, event): + with open('/etc/example_blog/mode') as mode_file: + prev_mode = mode_file.read().strip() + if self.model.config['mode'] == prev_mode: + return + + with open('/etc/example_blog/mode', 'w') as mode_file: + mode_file.write(f'{mode}\n') + + self._restart() +``` + +One common scenario where charm authors get tempted to use `StoredState`, when a no-op would be better, is to use `StoredState` to cache information from the Juju model. The Operator Framework already caches information about relations, unit and application names, etc. It reads and loads the charm's config into memory during each hook execution. Authors can simply fetch model and config information as needed, trusting that the Operator Framework is avoiding extra work where it can, and doing extra work to avoid cache coherency issues where it must. + +Another temptation is to track the occurence of certain events like [`pebble-ready`](https://juju.is/docs/sdk/events#heading--pebble-ready). This is dangerous. The emission of a `pebble-ready` event means that Pebble was up and running when the hook was invoked, but makes no guarantees about the future. Pebble may not remain running -- see the note about the Kubernetes scheduler above -- meaning your `StoredState` contains an invalid cache value which will likely lead to bugs. In cases where charm authors want to perform an action if and only if the workload container is up and running, they should guard against Pebble issues by catching `ops.pebble.ConnectionError`: + +```python +def some_event_handler(event): + try: + self.do_thing_that_assumes_container_running() + except ops.pebble.ConnectionError: + event.defer() + return +``` + +You shouldn't use the container's `can_connect()` method for the same reason - it's a point-in-time check, and Pebble could go away between calling `can_connect()` and when the actual change is executed - ie. you've introduced a race condition. + +In the other cases where state is needed, authors ideally want to relate a charm to a database, attach storage ([see Juju storage](https://juju.is/docs/sdk/storage)), or simply be opinionated, and hard code the single "correct" state into the charm. (Perhaps `ExampleBlog` should always be run in `production` mode when deployed as a charm?) + +In the cases where it is important to share some lightweight configuration data between units of an application, charm author's should look into [peer relations](https://juju.is/docs/sdk/integration#heading--peer-integrations). And in the cases where data must be written to a container's local file system (Canonical's Kubeflow bundle, for example, must do this, because the sheer number of services mean that we run into limitations on attached storage in the underlying cloud), authors should do so mindfully, with an understanding of the pitfalls involved. + +In sum: use state mindfully, with well chosen tools, only when necessary. \ No newline at end of file diff --git a/docs/howto/get-started-with-charm-testing.md b/docs/howto/get-started-with-charm-testing.md new file mode 100644 index 000000000..d947fc668 --- /dev/null +++ b/docs/howto/get-started-with-charm-testing.md @@ -0,0 +1,332 @@ +(get-started-with-charm-testing)= +# Get started with charm testing + +Testing charm code is an essential part of charming. Here we will see how to get started with it. We will look at the templates we have available and the frameworks we can use to write good unit, integration, and functional tests. + +**What you'll need:** +- knowledge of testing in general +- knowledge of Juju and charms +- knowledge of the Juju models and events, esp. the data involved in a charm's lifecycle (e.g. see {ref}`Talking to a workload control flow from A to Z `) + +**What you will learn:** +- What are the starting points for adding tests to a charm? +- What do you typically want to test in a charm? +- How can you do that? + - What can you unit-test? + - How to effectively use the Harness. + - What can you only integration-test? + - What integration testing frameworks are there? + - Where can you apply functional testing? +- How to automate this in a CI pipeline. + +## Charmcraft profiles + +The most popular way to set up a charm project is via `charmcraft init`. For example, to get set up for creating a machine charm, run: + +```text +charmcraft init --profile=machine +``` + +This will provide the following files that you'll use when writing your tests (as well as others for building your charm): + +```text +. +├── pyproject.toml +├── spread.yaml +├── tests +│ ├── integration +│ │ └── test_charm.py +│ ├── spread +│ │ ├── general +│ │ │ └── integration +│ │ │ └── task.yaml +│ │ └── lib +│ │ ├── cloud-config.yaml +│ │ └── test-helpers.sh +│ └── unit +│ └── test_charm.py +└── tox.ini +``` + +There are also profiles for `kubernetes` and for building charms for apps developed with popular frameworks such as Django and Flask. + +> See more: [ Write your first Kubernetes charm for a Flask app](https://juju.is/docs/sdk/write-your-first-kubernetes-charm-for-a-flask-app) + +## Unit testing + +### A charm as an input -> output function + +In production, a charm is an object that comes to life when the Juju agent decides to execute it with a given context (we call that an `event`). +The "inputs" of a charm run are therefore: + + - the event context + - charm configuration + - integration (relation) data + - stored state + +Only the event context is guaranteed to be present. The other input sources are optional, but typically a charm will have at least some config and a few integrations adding to its inputs. + +The charm code executes and typically produces side-effects aimed at its workload (for example: it writes files to a disk, runs commands on a system, or reconfigures a process) or at other charms it integrates with (for example: it writes relation data). We call this 'operating' a workload, and that is what a charm is meant to do. The ways in which a charm operates can be roughly categorised as: + +- system operations (e.g. kill a process, restart a service, write a file, emit to a syslog server, make a HTTP request) +- cloud operations (e.g. deploy a Kubernetes service, launch a VM) +- workload operations (e.g. send a request to a local server, write a config file) +- Juju operations (write relation data) + +If the charm is a machine charm, workload operation calls can be done directly, while if we're talking about a Kubernetes charm, they will likely be mediated by [Pebble](https://github.com/canonical/pebble). + +"Juju operations" are the most 'meta' of them all: they do not affect the workload in and of itself, but they share data which is meant to affect the operation of *other* charms that this charm is integrated with. + +> See more: {ref}`Talking to a workload: control flow from A to Z `, {ref}`Charm lifecycle ` + +### What we are testing when we unit-test + +A 'live', deployed Juju application will have access to all the inputs we discussed above, including environment variables, host system access, and more. Unit tests will typically want to mock all that and focus on mapping inputs to expected outputs. Any combination of the input sources we mentioned above can result in any combination of operations. A few examples of increasing complexity of scenarios we may want to unit test: + + - if this event occurs, assert that the charm emits that system call + - if this event occurs, given this config, assert that the charm writes to the filesystem a config file with this expected content + - if this event occurs, given this relation data and that config value, assert that that system call is made and this relation data is written (to another relation) + +You will notice that the starting point is typically always an event. A charm doesn't do anything unless it's being run, and it is only run when an event occurs. So there is *always* an event context to be mocked. This has important consequences for the unit-testing framework, as we will see below. + +### The harness + +In the charming world, unit testing means using Harness. + +> See more [`ops.testing.Harness`](https://ops.readthedocs.io/en/latest/harness.html#ops.testing.Harness) + +Harness is the 'mocker' for most inputs and outputs you will need. Where a live charm would gather its input through context variables and calls to the Juju api (by running the hook tools), a charm under unit test will gather data via a mocked backend managed by Harness. Where a live charm would produce output by writing files to a filesystem, Harness exposes a mock filesystem the charm will be able to interact with without knowing the difference. More specific outputs, however, will need to be mocked individually. + +A typical test with Harness will look like this: + +- set things up: + - set up the charm and its metadata + - set up the harness + - mock any 'output' callable that you know would misfire or break (e.g. a system call -- you don't want a unittest to reboot your laptop) + - configure the charm + - mock any relation data + - **mock an event** + - get the output + - run assertions on the output + +> Obviously, other flows are possible; for example, where you unit test individual charm methods without going through the whole event context setup, but this is the characteristic one. + +### Understanding Harness + +When you instantiate a `Harness` object, the charm instance does not exist yet. Just like in a live charm, it is possible that when the charm is executed for the first time, the Juju model already has given it storage, relations, some config, or leadership. This delay is meant to give us a chance to simulate this in our test setup. You create a `Harness` object, then you prepare the 'initial state' of the model mock, then you finally initialise the charm and simulate one or more events. + +There are two ways to initialize a harnessed charm: + + * When a charm is deployed, it goes through the Setup phase, a fixed sequence of events. `Harness` has a method, `begin_with_initial_hooks()`, that runs this sequence. + * Alternatively, you can initialise the charm by calling `begin()`. This will instantiate the charm without firing any Setup phase event. + +> See more: {ref}`A charm's life `, [`ops.testing.Harness.begin_with_initial_hooks()`](https://ops.readthedocs.io/en/latest/harness.html#ops.testing.Harness.begin_with_initial_hooks), [`ops.testing.Harness.begin()`](https://ops.readthedocs.io/en/latest/harness.html#ops.testing.Harness.begin) + +After the Setup phase, the charm goes into Operation. To test operation-phase-related events, the harness provides some methods to simulate the most common scenarios. For example: + + - the cloud admin changes the charm config: `harness.update_config` + - the cloud admin relates this charm to some other: `harness.add_relation` + - a remote unit joins in a relation (e.g. because the cloud admin has scaled up a remote charm): `harness.add_relation_unit` + - a remote unit touches its relation data: `harness.update_relation_data` + - the cloud admin removes a relation: `harness.remove_relation` + - a resource is attached/detached: `harness.attach_storage` / `harness.detach_storage` + - a container becomes ready: `harness.container_pebble_ready` + +Therefore, one typically will not have to manually `.emit()` events, but can rely on the `Harness` utilities and focus on the higher level abstractions that they expose. + +### Writing a test + +The typical way in which we want to structure a test is: + - configure the required inputs + - mock any function or system call you need to + - initialise the charm + - fire some event OR use one of the harness methods to trigger a predefined event sequence + - assert some output matches what is expected, or some function is called with the expected parameters, etc... + +A simple test might look like this: + +```python +from charm import MyCharm +from ops import testing + +def test_pebble_ready_writes_config_file(): + """Test that on pebble-ready, a config file is written""" + harness: testing.Harness[MyCharm] = testing.Harness(MyCharm) + # If you want to mock charm config: + harness.update_config({'foo': 'bar'}) + # If you want to mock charm leadership: + harness.set_leader(True) + + # If you want to mock relation data: + relation_ID = harness.add_relation('relation-name', 'remote-app-name') + harness.add_relation_unit(relation_ID, 'remote-app-name/0') + harness.update_relation_data(relation_ID, 'remote-app-name/0', {'baz': 'qux'}) + + # We are done setting up the inputs. + + harness.begin() + charm = harness.charm # This is a MyCharm instance. + + # This will fire a `-pebble-ready` event. + harness.container_pebble_ready("workload") + + # Suppose that MyCharm has written a YAML config file via Pebble.push(): + container = charm.unit.get_container("workload") + file = "/opt/workload/path_to_config_file.yaml" + config = yaml.safe_load(container.pull(file).read()) + assert config[0]['foo']['bar'] == 'baz' # or whatever +``` + +```{note} + +An important difference between a harnessed charm and a 'live', deployed, charm is that `Harness` holds on to the charm instance between events, while a deployed charm garbage-collects the charm instance between hooks. So if your charm were to set some states in, say, instance attributes, and rely on it on subsequent event handling loops, the unit tests based on the harness would not be able to catch that mistake. An integration test would. + +``` + +## Integration testing + +Where unit testing focuses on black-box method-by-method verification, integration testing focuses on the big picture. Typically integration tests check that the charm does not break (generally this means: blocks with status `blocked` or `error`) when a (mocked) cloud admin performs certain operations. These operations are scripted by using, in order of abstraction: + - shell commands against [the `juju` cli](https://juju.is/docs/olm/juju-cli-commands) + - [`python-libjuju`](https://github.com/juju/python-libjuju), wrapping juju api calls + - [`pytest-operator`](https://github.com/charmed-kubernetes/pytest-operator), a `pytest` plugin wrapping `python-libjuju` + - [`zaza`](https://zaza.readthedocs.io/en/latest/index.html), a testing-framework-agnostic wrapper on top of `python-libjuju` + +Things you typically want to test with integration tests: + - The charm can be packed (i.e. `charmcraft pack` does not error) + - The charm can be deployed (i.e. `juju deploy ./packed_charm.charm` deploys an application that reaches `active` or `waiting` within a reasonable time frame) + +These are 'smoke tests' that should always be present, and are provided for you when using `charmcraft init`. The following are non-smokey, proper integration tests. +- The charm can be related to other applications without erroring + - and the relation has the expected effect on the charm's operation logic +- The charm can be configured + - and the config has the expected effect on the charm's operation logic +- The actions supported by the charm can be executed + - and return the expected results +- Given any combination of the above relations, configs, etc, the charm 'works': the workload it operates does whatever it is supposed to do. + +### Testing with `pytest-operator` + +The integration test template that `charmcraft init` provides includes the starting point for writing integration tests with `pytest-operator` and `python-libjuju`. The `tox.ini` is also configured so that you can run `tox -e integration` to run the tests, provided that you have a cloud (such as LXD or microk8s) and local Juju client. + +The entry point for all `pytest-operator` tests is the `ops_test` fixture. The fixture is a module-scoped context which, on entry, adds to Juju a randomly-named new model and destroys it on exit. All tests in that module, and all interactions with the `ops_test` object, will take place against that model. + +Once you have used `ops_test` to get a model in which to run your integration tests, most of the remaining integration test code will interact with Juju via the `python-libjuju` package. + +> See more: [`python-libjuju`](https://pythonlibjuju.readthedocs.io/en/latest/) + +```{note} + +*Pro tip*: you can prevent `ops_test` from tearing down the model on exit by passing the `--keep-models` argument. This is useful when the tests fail and the logs don't provide a sufficient post-mortem and a real live autopsy is required. + +``` + +Detailed documentation of how to use `ops_test` and `pytest-operator` is out of scope for this document. However, this is an example of a typical integration test: + +```python +async def test_operation(ops_test: OpsTest): + # Tweak the config: + app: Application = ops_test.model.applications.get("tester") + await app.set_config({"my-key": "my-value"}) + + # Add another charm and relate them: + await ops_test.model.deploy('other-app') + await ops_test.model.relate('tester:endpoint1', 'other-charm:endpoint2') + + # Scale it up: + await app.add_unit(2) + + # Remove another app: + await ops_test.model.remove_application('yet-another-app') + + # Run an action on a unit: + unit: Unit = app.units[1] + action = await unit.run('my-action') + assert action.results == + + # What this means depends on the workload: + assert charm_operates_correctly() +``` + +`python-libjuju` has, of course, an API for all inverse operations: remove an app, scale it down, remove a relation... + +A good integration testing suite will check that the charm continues to operate as expected whenever possible, by combining these simple elements. + +## Functional testing + +Some charms represent their workload by means of an object-oriented wrapper, which mediates between operator code and the implementation of operation logic. In such cases, it can be useful to add a third category of tests, namely functional tests, that black-box test that workload wrapper without worrying about the substrate it runs on (the charm, the cloud, the machine or pod...). +For an example charm adopting this strategy, see [parca-operator](https://github.com/jnsgruk/parca-operator). Nowadays, the preferred tool to do functional testing is Scenario. + +> See more: [Scenario](https://github.com/canonical/ops-scenario), {ref}`Write a Scenario test for a charm ` + +## Continuous integration + +Typically, you want the tests to be run automatically against any PR into your repository's main branch, and sometimes, to trigger a new release whenever that succeeds. CD is out of scope for this article, but we will look at how to set up a basic CI. + +Create a file called `.github/workflows/ci.yaml`. For example, to include a `lint` job that runs the `tox` `lint` environment: + +```yaml +name: Tests +on: + workflow_call: + +jobs: + lint: + name: Lint + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v3 + - name: Install dependencies + run: python3 -m pip install tox + - name: Run linters + run: tox -e lint +``` + +Other `tox` environments can be run similarly; for example unit tests: + +```yaml + unit-test: + name: Unit tests + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v3 + - name: Install dependencies + run: python -m pip install tox + - name: Run tests + run: tox -e unit +``` + +Integration tests are a bit more complex, because in order to run those tests, a Juju controller and a cloud in which to deploy it, is required. This example uses a `actions-operator` workflow provided by `charmed-kubernetes` in order to set up `microk8s` and Juju: + +``` + integration-test-microk8s: + name: Integration tests (microk8s) + needs: + - lint + - unit-test + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v3 + - name: Setup operator environment + uses: charmed-kubernetes/actions-operator@main + with: + provider: microk8s + - name: Run integration tests + # Set a predictable model name so it can be consumed by charm-logdump-action + run: tox -e integration -- --model testing + - name: Dump logs + uses: canonical/charm-logdump-action@main + if: failure() + with: + app: my-app-name + model: testing +``` + +> You can find more actions, advanced documentation and use cases in [charming-actions](https://github.com/canonical/charming-actions) + +## Conclusion + +We have examined all angles one might take when testing a charm, and given a brief overview of the most popular frameworks for implementing unit and integration tests, all the way to how one would link them up with a CI system to make sure the repository remains clean and tested. + diff --git a/docs/howto/index.md b/docs/howto/index.md new file mode 100644 index 000000000..8fec3f173 --- /dev/null +++ b/docs/howto/index.md @@ -0,0 +1,29 @@ +(howto-guides)= +# How-to guides + +```{toctree} +:maxdepth: 2 + +Manage logs +Run workloads with a charm machines +Run workloads with a charm Kubernetes +Manage storage +Manage resources +Manage actions +Manage configurations +Manage relations +Manage leadership changes +Manage libraries +Manage interfaces +Manage secrets +Manage the charm version +Manage the workload version +Get started with charm testing +Write unit tests for a charm +Write scenario tests for a charm +Write integration tests for a charm +Instrument your charm with tracing telemetry +Turn a hooks-based charm into an ops charm + +``` + diff --git a/docs/howto/instrument-your-charm-with-tracing-telemetry.md b/docs/howto/instrument-your-charm-with-tracing-telemetry.md new file mode 100644 index 000000000..ef8f4d607 --- /dev/null +++ b/docs/howto/instrument-your-charm-with-tracing-telemetry.md @@ -0,0 +1,146 @@ +(how-to-instrument-your-charm-with-tracing-telemetry)= +# How to instrument your charm with tracing telemetry + +```{note} + +This howto has been incorporated into [the Tempo charm docs](https://charmhub.io/topics/charmed-tempo-ha/instrument%20your%20charm). + +``` + + +In order to instrument a charm with tracing telemetry, you will need to: + +1. Set up a model with `cos-lite`: + + +```bash +juju add-model clite +juju deploy cos-lite --trust +``` + +> See more: [Charmhub | `cos-lite`](https://charmhub.io/cos-lite) + +2. Deploy `tempo`: + +```bash +juju deploy tempo-k8s +``` + +> See more: [Charmhub | `tempo-k8s`](https://charmhub.io/tempo-k8s) + +3. Integrate tempo with `cos-lite`: + +```bash +jhack imatrix fill +``` + +> See more: {ref}``jhack` `, [GitHub | `jhack` > `imatrix` > `fill`](https://github.com/PietroPasotti/jhack#fill) + +Alternatively, you can integrate manually by: +``` +juju integrate tempo:logging loki:logging +juju integrate tempo:ingress traefik:ingress +juju integrate grafana:grafana-source tempo:grafana-source +juju integrate prometheus:metrics-endpoint, tempo:metrics-endpoint +juju integrate grafana:grafana-dashboard, tempo:grafana-dashboard +juju integrate traefik:tracing, tempo:tracing +juju integrate prometheus:tracing, tempo:tracing +``` + +```{note} + At some point there will be an overlay bundle to deploy `cos-lite` + `tempo` and integrate them; see [this PR](https://github.com/canonical/cos-lite-bundle/pull/79) to follow the progress on the overlay. +``` + +At this point you should have a working `cos-lite` plus `tempo` deployment; something like this: +![image|690x333](upload://7ohHj2JBT9Zaezsgx6p3a56n5cO.png) + +4. Fetch the `charm_tracing` and the `tracing` libs: +```bash +charmcraft fetch-lib charms.tempo-k8s.v1.charm_tracing +charmcraft fetch-lib charms.tempo-k8s.v2.tracing +``` + +> See more: {ref}``charmcraft fetch-lib` `, [Charmhub | `tempo-k8s` > Libraries > `charm-tracing`](https://charmhub.io/tempo-k8s/libraries/charm_tracing), [Charmhub | `tempo-k8s` > Libraries > `tracing`](https://charmhub.io/tempo-k8s/libraries/tracing) + +5. Add opentelemetry exporter dependency used by `charm_tracing` to your `requirements.txt`: + +``` +opentelemetry-exporter-otlp-proto-http>=1.21.0 +``` + +6. Add an integration with `tempo`: + +```yaml +# in charmcraft.yaml +provides: + tracing: + interface: tracing + limit: 1 +``` + +> See more: {ref}`File `charmcraft.yaml` > `provides` <11012md>` + +7. Instrument your charm code: + +```python +# in /your/charm/project/src/charm.py +from charms.tempo_k8s.v1.charm_tracing import trace_charm +from charms.tempo_k8s.v2.tracing import TracingEndpointRequirer + +@trace_charm(tracing_endpoint="tracing_endpoint") +class MyCharm(CharmBase): + def __init__(self, *args): + super().__init__(*args) + # add a provider wrapper for the tracing endpoint + self.tracing = TracingEndpointRequirer(self, protocols={ref}`"otlp_http"]) + + @property + def tracing_endpoint(self) -> Optional[str]: + """Tempo endpoint for charm tracing.""" + if self.tracing.is_ready(): + return self.tracing.get_endpoint("otlp_http") + return None +``` +```{note} + +By default, the traces generated by this charm will be tagged with service name equal to the class name of the charm, so `"MyCharm"` in this case. You override this default by passing +`service_name="foo"` to `trace_charm`. + +``` + + +8. Pack, deploy, and integrate your charm with `cos-lite`: + +```bash +charmcraft pack +juju deploy ./my-charm-operator_ubuntu-20.04-amd64.charm mycharm \ + $(yq eval '.resources | to_entries | map(select(.value.upstream-source != null) | "--resource " + .key + "=" + .value.upstream-source) | join(" ")' charmcraft.yaml) +juju integrate mycharm:tracing tempo-k8s:tracing +``` + +```{note} + +`cos-lite` is a kubernetes bundle. `tempo` is a Kubernetes charm. + +``` + +> See more: [`charmcraft pack` `, [Juju | `juju deploy`](https://juju.is/docs/juju/juju-deploy), [Juju | `juju integrate`](https://juju.is/docs/juju/juju-integrate) + +9. View the traces for the charm. + +Open the Grafana dashboard in a browser ([see here](https://github.com/canonical/grafana-k8s-operator) for more detailed instructions). + +Next, navigate to the traces for your charm: +- go to `Explore` and select the Tempo datasource. +- pick the `service_name` you gave to MyCharm above (the default is the application name) to see the traces for that charm +![image|690x505](upload://rAZOurgeYtFfymdB1V0qzKg7HcW.png) +- click on a trace ID to visualize it. For example, this is the trace for an `update-status` event on the Tempo charm itself: +![image|690x492](upload://6lufy3oZUYDkwxrnufQJKf5tpGN.png) + + +## Mapping events to traces with jhack tail +`jhack tail` supports a `-t` option to show the trace IDs associated with a charm execution: + +![image|607x326](upload://1UUxRnTkBR3hkCPfWd0sd4AXqfJ.png) + +This means that you can tail a charm, grab the trace id from tail, put it in the grafana dashboard query and get to the trace in no time. diff --git a/docs/howto/manage-actions.md b/docs/howto/manage-actions.md new file mode 100644 index 000000000..530878114 --- /dev/null +++ b/docs/howto/manage-actions.md @@ -0,0 +1,215 @@ +(manage-actions)= +# How to manage actions + +> See first: [`juju` | Action](https://juju.is/docs/juju/action), [`juju` | Manage actions](https://juju.is/docs/juju/manage-actions), [`charmcraft` | Manage actions]() + + +## Implement the feature + +### Declare the action in `charmcraft.yaml` + +To tell users what actions can be performed on the charm, define an `actions` section in `charmcraft.yaml` that lists the actions and information about each action. The actions should include a short description that explains what running the action will do. Normally, all parameters that can be passed to the action are also included here, including the type of parameter and any default value. You can also specify that some parameters are required when the action is run. +For example: +```yaml +actions: + snapshot: + description: Take a snapshot of the database. + params: + filename: + type: string + description: The name of the snapshot file. + compression: + type: object + description: The type of compression to use. + properties: + kind: + type: string + enum: + - gzip + - bzip2 + - xz + default: gzip + quality: + description: Compression quality + type: integer + default: 5 + minimum: 0 + maximum: 9 + required: + - filename + additionalProperties: false +``` + + + +### Observe the action event and define an event handler + +In the `src/charm.py` file, in the `__init__` function of your charm, set up an observer for the action event associated with your action and pair that with an event handler. For example: + +``` +self.framework.observe(self.on.grant_admin_role_action, self._on_grant_admin_role_action) +``` + + +Now, in the body of the charm definition, define the action event handler. For example: + +``` +def _on_grant_admin_role_action(self, event): + """Handle the grant-admin-role action.""" + # Fetch the user parameter from the ActionEvent params dict + user = event.params["user"] + # Do something useful with it + cmd = ["/usr/bin/myapp", "roles", "system_admin", user] + # Set a log message for the action + event.log(f"Running this command: {' '.join(cmd)}") + granted = subprocess.run(cmd, capture_output=True) + if granted.returncode != 0: + # Fail the action if there is an error + event.fail( + f"Failed to run '{' '.join(cmd)}'. Output was:\n{granted.stderr.decode('utf-8')}" + ) + else: + # Set the results of the action + msg = f"Ran grant-admin-role for user '{user}'" + event.set_results({"result": msg}) +``` + +More detail below: + + +#### Use action params + +To make use of action parameters, either ones that the user has explicitly passed, or default values, use the `params` attribute of the event object that is passed to the handler. This is a dictionary of parameter name (string) to parameter value. For example: +```python +def _on_snapshot(self, event: ops.ActionEvent): + filename = event.params["filename"] + … +``` +> See more: [`ops.ActionEvent.params`](https://ops.readthedocs.io/en/latest/#ops.ActionEvent.params) + + +#### Report that an action has failed + +To report that an action has failed, in the event handler definition, use the fail() method along with a message explaining the failure to be shown to the person running the action. Note that the `fail()` method doesn’t interrupt code execution, so you will usually want to immediately follow the call to `fail()` with a `return`, rather than continue with the event handler. For example: +```python +def _on_snapshot(self, event: ops.ActionEvent): + filename = event.params['filename'] + kind = event.params['compression']['kind'] + quality = event.params['compression']['quality'] + cmd = ['/usr/bin/do-snapshot', f'--kind={kind}', f'--quality={quality}', filename] + subprocess.run(cmd, capture_output=True) + if granted.returncode != 0: + event.fail( + f"Failed to run {' '.join(cmd)!r}. Output was:\n{granted.stderr.decode('utf-8')}" + ) + … +``` +] +> See more: [`ops.ActionEvent.fail`](https://ops.readthedocs.io/en/latest/#ops.ActionEvent.fail) + +#### Return the results of an action + +To pass back the results of an action to the user, use the `set_results` method of the action event. These will be displayed in the `juju run` output. For example: +```python +def _on_snapshot(self, event: ops.ActionEvent): + size = self.do_snapshot(event.params['filename']) + event.set_results({'snapshot-size': size}) +``` +> See more: [`ops.ActionEvent.set_results`](https://ops.readthedocs.io/en/latest/#ops.ActionEvent.set_results) + + +#### Log the progress of an action + +In a long-running action, to give the user updates on progress, use the `.log()` method of the action event. This is sent back to the user, via Juju, in real-time, and appears in the output of the `juju run` command. For example: +```python +def _on_snapshot(self, event: ops.ActionEvent): + event.log('Starting snapshot') + self.snapshot_table1() + event.log('Table1 complete') + self.snapshot_table2() + event.log('Table2 complete') + self.snapshot_table3() +``` +> See more: [`ops.ActionEvent.log`](https://ops.readthedocs.io/en/latest/#ops.ActionEvent.log) + + +#### Record the ID of an action task + +When a unique ID is needed for the action task - for example, for logging or creating temporary files, use the `.id` attribute of the action event. For example: +```python +def _on_snapshot(self, event: ops.ActionEvent): + temp_filename = f'backup-{event.id}.tar.gz' + logger.info("Using %s as the temporary backup filename in task %s", filename, event.id) + self.create_backup(temp_filename) + ... +``` +> See more: [`ops.ActionEvent.id`](https://ops.readthedocs.io/en/latest/#ops.ActionEvent.id) + + +## Test the feature + +> See first: {ref}`get-started-with-charm-testing` + +What you need to do depends on what kind of tests you want to write. + + +### Write unit tests + +> See first: {ref}`write-unit-tests-for-a-charm` + +When using Harness for unit tests, use the `run_action` method to verify that charm actions have the expected behaviour. This method will either raise an `ActionFailed` exception (if the charm used the `event.fail()` method) or return an `ActionOutput` object. These can be used to verify the failure message, logs, and results of the action. For example: + +```python +def test_backup_action(): + harness = ops.testing.Harness() + harness.begin() + try: + out = harness.run_action('snapshot', {'filename': 'db-snapshot.tar.gz'}) + except ops.testing.ActionFailed as e: + assert "Could not backup because" in e.message + else: + assert out.logs == ['Starting snapshot', 'Table1 complete', 'Table2 complete'] + assert 'snapshot-size' in out.results + finally: + harness.cleanup() + + +``` + +> See more: [`ops.testing.Harness.run_action`](https://ops.readthedocs.io/en/latest/#ops.testing.Harness.run_action) + + +### Write scenario tests +> See first: {ref}`write-scenario-tests-for-a-charm` + +When using Scenario for unit tests, to verify that the charm state is as expected after executing an action, use the `run_action` method of the Scenario `Context` object. The method returns an `ActionOutput` object that contains any logs and results that the charm set. +For example: +```python +def test_backup_action(): + action = scenario.Action('snapshot', params={'filename': 'db-snapshot.tar.gz'}) + ctx = scenario.Context(MyCharm) + out = ctx.run_action(action, scenario.State()) + assert out.logs == ['Starting snapshot', 'Table1 complete', 'Table2 complete'] + if out.success: + assert 'snapshot-size' in out.results + else: + assert 'Failed to run' in out.failure +``` +> See more: [Scenario action testing](https://github.com/canonical/ops-scenario?tab=readme-ov-file#actions) + + +### Write integration tests +> See first: {ref}`write-integration-tests-for-a-charm` + +To verify that an action works correctly against a real Juju instance, write an integration test with `pytest_operator`. For example: + +```python +async def test_logger(ops_test): + app = ops_test.model.applications[APP_NAME] + unit = app.units[0] # Run the action against the first unit. + action = await unit.run_action('snapshot', filename='db-snapshot.tar.gz') + action = await action.wait() + assert action.status == 'completed' + assert action.results['snapshot-size'].isdigit() +``` + diff --git a/docs/howto/manage-configurations.md b/docs/howto/manage-configurations.md new file mode 100644 index 000000000..aa788d44e --- /dev/null +++ b/docs/howto/manage-configurations.md @@ -0,0 +1,122 @@ +(manage-configurations)= +# Manage configurations + +> See first: [`juju` | Application configuration](https://juju.is/docs/juju/configuration#heading--application-configuration), [`juju` | Manage application configurations](https://juju.is/docs/juju/manage-applications#heading--configure-an-application), [`charmcraft` | Manage configurations]() + + +## Implement the feature + +### Define a configuration option + +In the `charmcraft.yaml` file of the charm, under `config.options`, add a configuration definition, including a name, a description, the type, and the default value. The example below shows how to define two configuration options, one called `name` of type `string` and default value `Wiki`, and one called `skin` with type `string` and default value `vector`: + +```text +config: + options: + name: + default: Wiki + description: The name, or Title of the Wiki + type: string + skin: + default: vector + description: skin for the Wiki + type: string +``` + + +### Observe the `config-changed` event and define the event handler + +In the `src/charm.py` file of the charm project, in the `__init__` function of the charm, set up an observer for the config changed event and pair that with an event handler: + +```python +self.framework.observe(self.on.config_changed, self._on_config_changed) +``` + +Then, in the body of the charm definition, define the event handler. Here you may want to read the current configuration value, validate it (Juju only checks that the *type* is valid), and log it, among other things. Sample code for an option called `server-port`, with type `int`, and default value `8000`: + + ```python +def _on_config_changed(self, event): + port = self.config["server-port"] + + if port == 22: + self.unit.status = ops.BlockedStatus("invalid port number, 22 is reserved for SSH") + return + + logger.debug("New application port is requested: %s", port) + self._update_layer_and_restart(None) +``` +> See more: [`ops.CharmBase.config`](https://ops.readthedocs.io/en/latest/#ops.CharmBase.config) + +```{caution} + + - Multiple configuration values can be changed at one time through Juju, resulting in only one `config_changed` event. Thus, your charm code must be able to process more than one config value changing at a time. +- If `juju config` is run with values the same as the current configuration, the `config_changed` event will not run. Therefore, if you have a single config value, there is no point in tracking its previous value -- the event will only be triggered if the value changes. +- Configuration cannot be changed from within the charm code. Charms, by design, aren't able to mutate their own configuration by themselves (e.g., in order to ignore an admin-provided configuration), or to configure other applications. In Ops, one typically interacts with config via a read-only facade. + +``` + +### (If applicable) Update and restart the Pebble layer + +**If your charm is a Kubernetes charm and the config affects the workload:** Update the Pebble layer to fetch the current configuration value and then restart the Pebble layer. + + + +## Test the feature + +> See first: {ref}`get-started-with-charm-testing` + +You'll want to add two levels of tests: unit and scenario. + +### Write unit tests + +> See first: {ref}`write-unit-tests-for-a-charm` + +To use a unit test to verify that the configuration change is handled correct, the test needs to trigger the `config-changed` event and then check that the update method was called. In your `tests/unit/test_charm.py` file, add the following test functions to the file: + +```python +def test_invalid_port_configuration(): + harness = ops.testing.Harness() + harness.begin() + + harness.update_config({"server-port": 22}) + assert isinstance(harness.model.unit.status, ops.BlockedStatus) + +def test_port_configuration(monkeypatch): + update_called = False + def mock_update(*args): + update_called = True + monkeypatch.setattr(MyCharm, "_update_layer_and_restart", mock_update) + + harness = ops.testing.Harness() + harness.begin() + + harness.update_config({"server-port": 8080}) + + assert update_called +``` + +### Write scenario tests + +> See first: {ref}`write-scenario-tests-for-a-charm` + +To use a Scenario test to verify that the `config-changed` event validates the port, pass the new config to the `State`, and, after running the event, check the unit status. For example, in your `tests/scenario/test_charm.py` file, add the following test function: + +```python +def test_open_port(): + ctx = scenario.Context(MyCharm) + + state_out = ctx.run("config_changed", scenario.State(config={"server-port": 22})) + + assert isinstance(state_out.unit_status, ops.BlockedStatus) +``` + +### Test-deploy + +To verify that the configuration option works as intended, pack your charm, update it in the Juju model, and run `juju config` followed by the name of the application deployed by your charm and then your newly defined configuration option key set to some value. For example, given the `server-port` key defined above, you could try: + +```text +juju config server-port=4000 +``` + diff --git a/docs/howto/manage-interfaces.md b/docs/howto/manage-interfaces.md new file mode 100644 index 000000000..1016554ee --- /dev/null +++ b/docs/howto/manage-interfaces.md @@ -0,0 +1,482 @@ +(manage-interfaces)= +# How to manage interfaces + +(register-an-interface)= +## Register an interface + + +Suppose you have determined that you need to create a new relation interface called `my_fancy_database`. + +Suppose that your interface specification has the following data model: +- the requirer app is supposed to forward a list of tables that it wants to be provisioned by the database provider +- the provider app (the database) at that point will reply with an API endpoint and, for each replica, it will provide a separate secret ID to authenticate the requests + +These are the steps you need to take in order to register it with {ref}``charm-relation-interfaces` `. + + +```{dropdown} Expand to preview some example results + +- [A bare-minimum example](https://github.com/IronCore864/charm-relation-interfaces/tree/my-fancy-database/interfaces/my_fancy_database/v0) +- [A more realistic example](https://github.com/canonical/charm-relation-interfaces/tree/main/interfaces/ingress/v1): + - As you can see from the [`interface.yaml`](https://github.com/canonical/charm-relation-interfaces/blob/main/interfaces/ingress/v1/interface.yaml) file, the [`canonical/traefik-k8s-operator` charm](https://github.com/canonical/traefik-k8s-operator) plays the provider role in the interface. + - The schema of this interface is defined in [`schema.py`](https://github.com/canonical/charm-relation-interfaces/blob/main/interfaces/ingress/v1/schema.py). + - You can find out more information about this interface in the [README](https://github.com/canonical/charm-relation-interfaces/blob/main/interfaces/ingress/v1/README.md). + + +``` + + +### 1. Clone (a fork of) [the `charm-relation-interfaces` repo](https://github.com/canonical/charm-relation-interfaces) and set up an interface specification folder + +```bash +git clone https://github.com/canonical/charm-relation-interfaces +cd /path/to/charm-relation-interfaces +``` + +### 2. Make a copy of the template folder +Copy the template folder to a new folder called the same as your interface (with underscores instead of dashes). + +```bash +cp -r ./interfaces/__template__ ./interfaces/my_fancy_database +``` + +At this point you should see this directory structure: + +``` +# tree ./interfaces/my_fancy_database +./interfaces/my_fancy_database +└── v0 + ├── README.md + ├── interface.yaml + ├── interface_tests + └── schema.py +2 directories, 3 files +``` + +### 3. Edit `interface.yaml` + +Add to `interface.yaml` the charm that owns the reference implementation of the `my_fancy_database` interface. Assuming your `my_fancy_database_charm` plays the `provider` role in the interface, your `interface.yaml` will look like this: + +```yaml +# interface.yaml +providers: + - name: my-fancy-database-operator # same as metadata.yaml's .name + url: https://github.com/your-github-slug/my-fancy-database-operator +``` + +### 4. Edit `schema.py` + +Edit `schema.py` to contain: + +```python +# schema.py + +from interface_tester.schema_base import DataBagSchema +from pydantic import BaseModel, AnyHttpUrl, Field, Json +import typing + + +class ProviderUnitData(BaseModel): + secret_id: str = Field( + description="Secret ID for the key you need in order to query this unit.", + title="Query key secret ID", + examples=["secret:12312323112313123213"], + ) + + +class ProviderAppData(BaseModel): + api_endpoint: AnyHttpUrl = Field( + description="URL to the database's endpoint.", + title="Endpoint API address", + examples=["https://example.com/v1/query"], + ) + + +class ProviderSchema(DataBagSchema): + app: ProviderAppData + unit: ProviderUnitData + + +class RequirerAppData(BaseModel): + tables: Json[typing.List[str]] = Field( + description="Tables that the requirer application needs.", + title="Requested tables.", + examples=[["users", "passwords"]], + ) + + +class RequirerSchema(DataBagSchema): + app: RequirerAppData + # we can omit `unit` because the requirer makes no use of the unit databags +``` + +To verify that things work as they should, you can `pip install pytest-interface-tester` and then run `interface_tester discover --include my_fancy_database` from the `charm-relation-interfaces` root. + +You should see: + +```yaml +- my_fancy_database: + - v0: + - provider: + - + - schema OK + - charms: + - my_fancy_database_charm (https://github.com/your-github-slug/my-fancy-database-operator) custom_test_setup=no + - requirer: + - + - schema OK + - +``` + +In particular pay attention to `schema`. If it says `NOT OK` then there is something wrong with the pydantic model. + +### 5. Edit `README.md` + +Edit the `README.md` file to contain: + +```markdown +# `my_fancy_database` + +## Overview +This relation interface describes the expected behavior between of any charm claiming to be able to interface with a Fancy Database and the Fancy Database itself. +Other Fancy Database-compatible providers can be used interchangeably as well. + +## Usage + +Typically, you can use the implementation of this interface from [this charm library](https://github.com/your_org/my_fancy_database_operator/blob/main/lib/charms/my_fancy_database/v0/fancy.py), although charm developers are free to provide alternative libraries as long as they comply with this interface specification. + +## Direction +The `my_fancy_database` interface implements a provider/requirer pattern. +The requirer is a charm that wishes to act as a Fancy Database Service consumer, and the provider is a charm exposing a Fancy Database (-compatible API). + +/```mermaid +flowchart TD + Requirer -- tables --> Provider + Provider -- endpoint, access_keys --> Requirer +/``` + +## Behavior + +The requirer and the provider must adhere to a certain set of criteria to be considered compatible with the interface. + +### Requirer + +- Is expected to publish a list of tables in the application databag + + +### Provide + +- Is expected to publish an endpoint URL in the application databag +- Is expected to create and grant a Juju Secret containing the access key for each shard and publish its secret ID in the unit databags. + +## Relation Data + +See the {ref}`\[Pydantic Schema\] <12689md>` + + +### Requirer + +The requirer publishes a list of tables to be created, as a json-encoded list of strings. + +#### Example +\```yaml +application_data: { + "tables": "{ref}`'users', 'passwords']" +} +\``` + +### Provider + +The provider publishes an endpoint url and access keys for each shard. + +#### Example +\``` +application_data: { + "api_endpoint": "https://foo.com/query" +}, +units_data : { + "my_fancy_unit/0": { + "secret_id": "secret:12312321321312312332312323" + }, + "my_fancy_unit/1": { + "secret_id": "secret:45646545645645645646545456" + } +} +\``` +``` + +### 6. Add interface tests + +> See more: {ref}`write-tests-for-an-interface` + +### 7. Open a PR to [the `charm-relation-interfaces` repo](https://github.com/canonical/charm-relation-interfaces) + +Finally, open a pull request to the `charm-relation-interfaces` repo and drive it to completion, addressing any feedback or concerns that the maintainers may have. + + +(write-tests-for-an-interface)= +## Write tests for an interface + +> See also: {ref}`interface-tests` + +Suppose you have an interface specification in {ref}`charm-relation-interfaces`, or you are working on one, and you want to add interface tests. These are the steps you need to take. + +We will continue from the running example from {ref}`register-an-interface`. Your starting setup should look like this: + +```text +$ tree ./interfaces/my_fancy_database +./interfaces/my_fancy_database +└── v0 + ├── interface.yaml + ├── interface_tests + ├── README.md + └── schema.py + +2 directories, 3 files +``` + + +### Create the test module + +Add a file to the `interface_tests` directory called `test_provider.py`. +> touch ./interfaces/my_fancy_database/interface_tests/test_provider.py + +### Write a test for the 'negative' path + +Write to `test_provider.py` the code below: + +```python +from interface_tester import Tester +from scenario import State, Relation + + +def test_nothing_happens_if_remote_empty(): + # GIVEN that the remote end has not published any tables + t = Tester( + State( + leader=True, + relations={ref}` + Relation( + endpoint="my-fancy-database", # the name doesn't matter + interface="my_fancy_database", + ) + ], + ) + ) + # WHEN the database charm receives a relation-joined event + state_out = t.run("my-fancy-database-relation-joined") + # THEN no data is published to the (local) databags + t.assert_relation_data_empty() +``` + +This test verifies part of a 'negative' path: it verifies that if the remote end did not (yet) comply with his part of the contract, then our side did not either. + +### Write a test for the 'positive' path + +Append to `test_provider.py` the code below: + +```python +import json + +from interface_tester import Tester +from scenario import State, Relation + + +def test_contract_happy_path(): + # GIVEN that the remote end has requested tables in the right format + tables_json = json.dumps(["users", "passwords"]) + t = Tester( + State( + leader=True, + relations=[ + Relation( + endpoint="my-fancy-database", # the name doesn't matter + interface="my_fancy_database", + remote_app_data={"tables": tables_json}, + ) + ], + ) + ) + # WHEN the database charm receives a relation-changed event + state_out = t.run("my-fancy-database-relation-changed") + # THEN the schema is satisfied (the database charm published all required fields) + t.assert_schema_valid() +``` + +This test verifies that the databags of the 'my-fancy-database' relation are valid according to the pydantic schema you have specified in `schema.py`. + +To check that things work as they should, you can run `interface_tester discover --include my_fancy_database` from the `charm-relation-interfaces` root. + +```{note} + +Note that the `interface_tester` is installed in the previous how-to guide [How to register an interface `. If you haven't done it yet, install it by running: `pip install pytest-interface-tester `. + +``` + +You should see: + +```yaml +- my_fancy_database: + - v0: + - provider: + - test_contract_happy_path + - test_nothing_happens_if_remote_empty + - schema OK + - charms: + - my_fancy_database_charm (https://github.com/your-github-slug/my-fancy-database-operator) custom_test_setup=no + - requirer: + - + - schema OK + - +``` + +In particular, pay attention to the `provider` field. If it says `` then there is something wrong with your setup, and the collector isn't able to find your test or identify it as a valid test. + +Similarly, you can add tests for requirer in `./interfaces/my_fancy_database/v0/interface_tests/test_requirer.py`. Don't forget to edit the `interface.yaml` file in the "requirers" section to add the name of the charm and the URL. See the "Edit `interface.yaml`" section in the previous how-to guide {ref}`How to register an interface <12690md>` for more detail on editing `interface.yaml`. [Here](https://github.com/IronCore864/charm-relation-interfaces/tree/my-fancy-database/interfaces/my_fancy_database/v0) is an example of tests for requirers added. + +### Merge in charm-relation-interfaces + +You are ready to merge this files in the charm-relation-interfaces repository. Open a PR and drive it to completion. + +#### Prepare the charm + +In order to be testable by charm-relation-interfaces, the charm needs to expose and configure a fixture. + +```{note} + +This is because the `fancy-database` interface specification is only supported if the charm is well-configured and has leadership, since it will need to publish data to the application databag. +Also, interface tests are Scenario tests and as such they are mock-based: there is no cloud substrate running, no Juju, no real charm unit in the background. So you need to patch out all calls that cannot be mocked by Scenario, as well as provide enough mocks through State so that the charm is 'ready' to support the interface you are testing. + +``` + +Go to the Fancy Database charm repository root. + +```text +cd path/to/my-fancy-database-operator +``` + +Create a `conftest.py` file under `tests/interface`: + +> mkdir ./tests/interface +> touch ./tests/interface/conftest.py + +Write in `conftest.py`: + +```python +import pytest +from charm import MyFancyDatabaseCharm +from interface_tester import InterfaceTester +from scenario.state import State + + +@pytest.fixture +def interface_tester(interface_tester: InterfaceTester): + interface_tester.configure( + charm_type=MyFancyDatabaseCharm, + state_template=State( + leader=True, # we need leadership + ), + ) + # this fixture needs to yield (NOT RETURN!) interface_tester again + yield interface_tester +``` + +```{note} + +This fixture overrides a homonym pytest fixture that comes with `pytest-interface-tester`. + +``` + + +````{note} + +You can configure the fixture name, as well as its location, but that needs to happen in the `charm-relation-interfaces` repo. Example: +``` +providers: + - name: my-fancy-database-provider + url: YOUR_REPO_URL + test_setup: + location: tests/interface/conftest.py + identifier: database_tester +``` + +```` + + +#### Verifying the `interface_tester` configuration + +To verify that the fixture is good enough to pass the interface tests, run the `run_matrix.py` script from the `charm-relation-interfaces` repo: + +```bash +cd path/to/charm-relation-interfaces +python run_matrix.py --include my_fancy_database +``` + +If you run this test, unless you have already merged the interface tests PR to `charm-relation-interfaces`, it will fail with some error message telling you that it's failing to collect the tests for the interface, because by default, `pytest-interface-tester` will try to find tests in the `canonical/charm-relation-interfaces` repo's `main` branch. + +To run tests with a branch in your forked repo, run: + +```bash +cd path/to/my-forked/charm-relation-interfaces +python run_matrix.py --include my_fancy_database --repo https://github.com/your-github-slug/charm-relation-interfaces --branch my-fancy-database +``` + +```{note} + +In the above command, remember to replace `your-github-slug` to your own slug, change the repo name accordingly (if you have renamed the forked repo), and update the `my-fance-database` branch name from the above command to the branch that contains your tests. + +``` + +Now the tests should be collected and executed. You should get similar output to the following: + +```bash +INFO:root:Running tests for interface: my_fancy_database +INFO:root:Running tests for version: v0 +INFO:root:Running tests for role: provider + +... + ++++ Results +++ +{ + "my_fancy_database": { + "v0": { + "provider": { + "my-fancy-database-operator": true + }, + "requirer": { + "my-fancy-database-operator": true + } + } + } +} +``` + +For reference, [here](https://github.com/IronCore864/my-fancy-database-operator) is an example of a bare minimum `my-fancy-database-operator` charm to make the test pass. In the charm, application relation data and unit relation data are set according to our definition (see the beginning part of the previous how-to guide {ref}`How to register an interface `). + +### Troubleshooting and debugging the tests + +#### Your charm is missing some configurations/mocks + +Solution to this is to add the missing mocks/patches to the `interface_tester` fixture in `conftest.py`. +Essentially, you need to make it so that the charm runtime 'thinks' that everything is normal and ready to process and accept the interface you are testing. +This may mean mocking the presence and connectivity of a container, system calls, substrate API calls, and more. +If you have scenario or unittests in your codebase, you most likely already have all the necessary patches scattered around and it's a matter of collecting them. + +Remember that if you run your tests using `run_matrix.py` locally, in your troubleshooting you need to point `interface.yaml` to the branch where you commited your changes as `run_matrix` fetches the charm repositories in order to run the charms: + +```text +requirers: + - name: my-fancy-database-operator + url: https://my-fancy-database-operator-repo + branch: branch-with-my-conftest-changes +``` +Remember, however, to merge the changes first in the operator repository before merging the pull request to `charm-relation-interfaces`. + +> See more: +> +> [Here](https://github.com/IronCore864/my-fancy-database-operator) is a minimum charm that both provides and requires the `my_fancy_database` interface from this how-to guide and [this](https://github.com/IronCore864/my-fancy-database-operator/blob/main/tests/interface/conftest.py) is an example of the bare minimum of `conftest.py`. See the content of `test_provider.py` [here in a forked repo](https://github.com/IronCore864/charm-relation-interfaces/blob/my-fancy-database/interfaces/my_fancy_database/v0/interface_tests/test_provider.py) and `test_requirer.py` [here](https://github.com/IronCore864/charm-relation-interfaces/blob/my-fancy-database/interfaces/my_fancy_database/v0/interface_tests/test_requirer.py). +> +> For a more realistic reference, refer to the [`test_provider.py`](https://github.com/canonical/charm-relation-interfaces/blob/main/interfaces/ingress/v1/interface_tests/test_provider.py) for the ingress interface defined in the `charm-relation-interfaces` repository, and check out the [`traefik-k8s-operator` charm](https://github.com/canonical/traefik-k8s-operator) for its content of the [`conftest.py`](https://github.com/canonical/traefik-k8s-operator/blob/main/tests/interface/conftest.py) file. + + + diff --git a/docs/howto/manage-leadership-changes.md b/docs/howto/manage-leadership-changes.md new file mode 100644 index 000000000..8017ea40f --- /dev/null +++ b/docs/howto/manage-leadership-changes.md @@ -0,0 +1,149 @@ +(manage-leadership-changes)= +# Manage leadership changes + +> See first: [`juju` | Leader](https://juju.is/docs/juju/leader) + + +## Implement response to leadership changes + +### Observe the `leader-elected` event and define an event handler + +In the `src/charm.py` file, in the `__init__` function of your charm, set up an observer for the `leader-elected` event and pair that with an event handler. For example: + +```python +self.framework.observe(self.on.leader_elected, self._on_leader_elected) +``` + +> See more: [`ops.LeaderElectedEvent`](https://ops.readthedocs.io/en/latest/#ops.LeaderElectedEvent) + +Now, in the body of the charm definition, define the event handler. For example, the handler below will update a configuration file: + +```python +def _on_leader_elected(self, event: ops.LeaderElectedEvent): + self.reconfigure(leader=self.unit) +``` + +> Examples: [Tempo reconfiguring ingress on leadership change](https://github.com/canonical/tempo-k8s-operator/blob/3f94027b6173f436968a4736a1f2d89a1f17b2e1/src/charm.py#L263), [Kubeflow Dashboard using a holistic handler to configure on leadership change and other events](https://github.com/canonical/kubeflow-dashboard-operator/blob/02caa736a6ea8986b8cba23b63c08a12aaedb86c/src/charm.py#L82) + +To have the leader notify other units about leadership changes, change data in a peer relation. + +> See more: [Peer Relations](https://juju.is/docs/juju/relation#heading--peer) + +{ref}`note status="Use the peer relation rather than `leader-setting-changed`"] +In the past, this was done by observing a `leader-setting-changed` event, which is now deprecated. +[/note] + +Commonly, other event handlers will need to check for leadership. For example, +only the leader unit can change charm application secrets, so checks for +leadership are needed to guard against non-leaders. For example: + +```python +if self.unit.is_leader(): + secret = self.model.get_secret(label="my-label") + secret.set_content({"username": "user", "password": "pass"}) +``` + +Note that Juju guarantees leadership for only 30 seconds after a `leader-elected` +event or an `is-leader` check. If the charm code may run longer, then extra +`is_leader()` calls should be made to ensure that the unit is still the leader. + +## Test response to leadership changes + +> See first: {ref}`get-started-with-charm-testing` + + +### Write unit tests + +> See first: {ref}`write-unit-tests-for-a-charm` + +When using Harness for unit tests, use the `set_leader()` method to control whether the unit is the leader. For example, to verify that leadership change is handled correctly: + +```python +@pytest.fixture() +def harness(): + yield ops.testing.Harness(MyCharm) + harness.cleanup() + + +def test_new_leader(harness): + # Before the test, the unit is not leader. + harness.set_leader(False) + harness.begin() + # Simulate Juju electing the unit as leader. + harness.set_leader(True) + # Assert that it was handled correctly. + assert ... + + +def test_leader_sets_secrets(harness): + # The unit is the leader throughout the test, and no leader-elected event + # is emitted. + harness.set_leader(True) + harness.begin() + secret_id = harness.add_model_secret(APP_NAME, content={"secret": "sssh"}) + harness.update_config(secret_option=secret_id) + # Assert that the config-changed handler set additional secret metadata: + assert ... +``` + +> See more: [`ops.testing.Harness.set_leader`](https://ops.readthedocs.io/en/latest/harness.html#ops.testing.Harness.set_leader) + +## Write scenario tests + +> See first: {ref}`write-scenario-tests-for-a-charm` + +When using Scenario for unit tests, pass the leadership status to the `State`. For example: + +```python +class MyCharm(ops.CharmBase): + def __init__(self, framework): + super().__init__(framework) + framework.observe(self.on.start, self._on_start) + + def _on_start(self, _): + if self.unit.is_leader(): + self.unit.status = ops.ActiveStatus('I rule') + else: + self.unit.status = ops.ActiveStatus('I am ruled') + + +@pytest.mark.parametrize('leader', (True, False)) +def test_status_leader(leader): + ctx = scenario.Context(MyCharm, meta={"name": "foo"}) + out = ctx.run('start', scenario.State(leader=leader)) + assert out.unit_status == ops.ActiveStatus('I rule' if leader else 'I am ruled') +``` + + +## Write integration tests + +> See first: {ref}`write-integration-tests-for-a-charm` + +Juju is in sole control over which unit is the leader, so leadership changes are +not usually tested with integration tests. If this is required, then the test +needs to remove the leader unit (machine charms) or run `juju_stop_unit` in the +charm container (Kubernetes charms). The test then needs to wait up to 60 seconds +for Juju to elect a new leader. + +More commonly, an integration test might want to verify that leader and non-leader behaviour is +as expected. For example: + +```python +async def get_leader_unit(ops_test, app, model=None): + """Utility method to get the current leader unit.""" + leader_unit = None + if model is None: + model = ops_test.model + for unit in model.applications[app].units: + if await unit.is_leader_from_status(): + leader_unit = unit + break + + return leader_unit +``` + +> Examples: [Zookeeper testing upgrades](https://github.com/canonical/zookeeper-operator/blob/106f9c2cd9408a172b0e93f741d8c9f860c4c38e/tests/integration/test_upgrade.py#L22), [postgresql testing password rotation action](https://github.com/canonical/postgresql-k8s-operator/blob/62645caa89fd499c8de9ac3e5e9598b2ed22d619/tests/integration/test_password_rotation.py#L38) + +> See more: [`juju.unit.Unit.is_leader_from_status`](https://pythonlibjuju.readthedocs.io/en/latest/api/juju.unit.html#juju.unit.Unit.is_leader_from_status) + + diff --git a/docs/howto/manage-libraries.md b/docs/howto/manage-libraries.md new file mode 100644 index 000000000..b7ad88929 --- /dev/null +++ b/docs/howto/manage-libraries.md @@ -0,0 +1,225 @@ +(manage-libraries)= +# Manage libraries +> See first: [`juju` | Library](), [`charmcraft` | Manage libraries]() + + +## Write a library + +When you're writing libraries, instead of callbacks, you can use custom events; that'll result in a more `ops`-native-feeling API. A custom event is, from a technical standpoint, an EventBase subclass that can be emitted at any point throughout the charm's lifecycle. These events are therefore totally unknown to Juju. They are essentially charm-internal, and can be useful to abstract certain conditional workflows and wrap the toplevel Juju event so it can be observed independently. + +For example, suppose you have a charm lib wrapping a relation endpoint. The wrapper might want to check that the remote end has sent valid data and, if that is the case, communicate it to the charm. For example, suppose that you have a `DatabaseRequirer` object, and the charm using it is interested in knowing when the database is ready. The `DatabaseRequirer` then will be: + +```python +class DatabaseReadyEvent(ops.charm.EventBase): + """Event representing that the database is ready.""" + + +class DatabaseRequirerEvents(ops.framework.ObjectEvents): + """Container for Database Requirer events.""" + ready = ops.charm.EventSource(DatabaseReadyEvent) + +class DatabaseRequirer(ops.framework.Object): + on = DatabaseRequirerEvents() + + def __init__(self, charm, ...): + [...] + self.framework.observe(self.charm.on.database_relation_changed, self._on_db_changed) + + def _on_db_changed(self, e): + if [...]: # check remote end has sent valid data + self.on.ready.emit() +``` + + +## Write tests for a library with Scenario + +In this guide we will go through how to write Scenario tests for a charm library we are developing: + +`/lib/charms/my_charm/v0/my_lib.py` + +The intended behaviour of this library (requirer side) is to copy data from the provider app databags and collate it in the own application databag. +The requirer side library does not interact with any lifecycle event; it only listens to relation events. + +### Setup + +Assuming you have a library file already set up and ready to go (see `charmcraft create-lib` otherwise), you now need to + +`pip install ops-scenario` and create a test file in `/tests/scenario/test_my_lib.py` + + +### Base test + +```python +# `/tests/scenario/test_my_lib.py` +import pytest +import ops +from scenario import Context, State +from lib.charms.my_Charm.v0.my_lib import MyObject + +class MyTestCharm(ops.CharmBase): + META = { + "name": "my-charm" + } + def __init__(self, framework): + super().__init__(framework) + self.obj = MyObject(self) + framework.observe(self.on.start, self._on_start) + + def _on_start(self, _): + pass + + +@pytest.fixture +def context(): + return Context(MyTestCharm, meta=MyTestCharm.META) + +@pytest.mark.parametrize('event', ( + 'start', 'install', 'stop', 'remove', 'update-status', #... +)) +def test_charm_runs(context, event): + """Verify that MyObject can initialize and process any event except relation events.""" + # arrange + state_in = State() + # act + context.run(event, state_in) +``` + +### Simple use cases + +#### Relation endpoint wrapper lib + +If `MyObject` is a relation endpoint wrapper such as [`traefik's ingress-per-unit`](https://github.com/canonical/traefik-k8s-operator/blob/main/lib/charms/traefik_k8s/v1/ingress_per_unit.py) lib, +a frequent pattern is to allow customizing the name of the endpoint that the object is wrapping. We can write a scenario test like so: + +```python +# `/tests/scenario/test_my_lib.py` +import pytest +import ops +from scenario import Context, State, Relation +from lib.charms.my_Charm.v0.my_lib import MyObject + + +@pytest.fixture(params=["foo", "bar"]) +def endpoint(request): + return request.param + + +@pytest.fixture +def my_charm_type(endpoint): + class MyTestCharm(ops.CharmBase): + META = { + "name": "my-charm", + "requires": + {endpoint: {"interface": "my_interface"}} + } + + def __init__(self, framework): + super().__init__(framework) + self.obj = MyObject(self, endpoint=endpoint) + framework.observe(self.on.start, self._on_start) + + def _on_start(self, _): + pass + + return MyTestCharm + + +@pytest.fixture +def context(my_charm_type): + return Context(my_charm_type, meta=my_charm_type.META) + + +def test_charm_runs(context): + """Verify that the charm executes regardless of how we name the requirer endpoint.""" + # arrange + state_in = State() + # act + context.run('start', state_in) + + +@pytest.mark.parametrize('n_relations', (1, 2, 7)) +def test_charm_runs_with_relations(context, endpoint, n_relations): + """Verify that the charm executes when there are one or more relations on the endpoint.""" + # arrange + state_in = State(relations=[ + Relation(endpoint=endpoint, interface='my-interface', remote_app_name=f"remote_{n}") for n in range(n_relations) + ]) + # act + state_out = context.run('start', state_in) + # assert + for relation in state_out.relations: + assert not relation.local_app_data # remote side didn't publish any data. + + +@pytest.mark.parametrize('n_relations', (1, 2, 7)) +def test_relation_changed_behaviour(context, endpoint, n_relations): + """Verify that the charm lib does what it should on relation changed.""" + # arrange + relations = [Relation( + endpoint=endpoint, interface='my-interface', remote_app_name=f"remote_{n}", + remote_app_data={"foo": f"my-data-{n}"} + ) for n in range(n_relations)] + state_in = State(relations=relations) + # act + state_out: State = context.run(relations[0].changed_event, state_in) + # assert + for relation in state_out.relations: + assert relation.local_app_data == {"collation": ';'.join(f"my-data-{n}" for n in range(n_relations))} +``` + +### Advanced use cases + +#### Testing internal (charm-facing) library APIs + +Suppose that `MyObject` has a `data` method that exposes to the charm a list containing the remote databag contents (the `my-data-N` we have seen above). +We can use `scenario.Context.manager` to run code within the lifetime of the Context like so: + +```python +import pytest +import ops +from scenario import Context, State, Relation +from lib.charms.my_Charm.v0.my_lib import MyObject + +@pytest.mark.parametrize('n_relations', (1, 2, 7)) +def test_my_object_data(context, endpoint, n_relations): + """Verify that the charm lib does what it should on relation changed.""" + # arrange + relations = [Relation( + endpoint=endpoint, interface='my-interface', remote_app_name=f"remote_{n}", + remote_app_data={"foo": f"my-data-{n}"} + ) for n in range(n_relations)] + state_in = State(relations=relations) + + with context.manager(relations[0].changed_event, state_in) as mgr: + # act + state_out = mgr.run() # this will emit the event on the charm + # control is handed back to us before ops is torn down + + # assert + charm = mgr.charm # the MyTestCharm instance ops is working with + obj: MyObject = charm.obj + assert obj.data == [ + f"my-data-{n}" for n in range(n_relations) + ] +``` + + +## Use a library + + +Fetch the library. + +> See more: [`charmcraft` | Manage libraries]() + +In your `src/charm.py`, observe the custom events it puts at your disposal. For example, a database library may have provided a `database_relation_ready` event -- a high-level wrapper around the relevant `juju` relation events -- so you use it to manage the database integration in your charm as below: + +```python + +class MyCharm(CharmBase): + def __init__(...): + [...] + self.db_requirer = DatabaseRequirer(self) + self.framework.observe(self.db_requirer.on.ready, self._on_db_ready) +``` + + diff --git a/docs/howto/manage-logs.md b/docs/howto/manage-logs.md new file mode 100644 index 000000000..2073b705e --- /dev/null +++ b/docs/howto/manage-logs.md @@ -0,0 +1,66 @@ +(how-to-log-a-message-in-a-charm)= +# How to log a message in a charm + +> See first: [`juju` | Log](https://juju.is/docs/juju/log), [`juju` | How to manage logs > Manage the logging configuration](https://juju.is/docs/juju/manage-logs#heading--manage-the-logging-configuration) + + + +To log a message in a charm, import Python's `logging` module, then use the `getLogger()` function with the desired level. For example: + + +```python +import logging +# ... +logger = logging.getLogger(__name__) + +class HelloOperatorCharm(ops.CharmBase): + # ... + + def _on_config_changed(self, _): + current = self.config["thing"] + if current not in self._stored.things: + # Note the use of the logger here + logger.debug("found a new thing: %r", current) + self._stored.things.append(current) +``` + + +> See more: +> - [`logging`](https://docs.python.org/3/library/logging.html), [`logging.getLogger()`](https://docs.python.org/3/library/logging.html#logging.getLogger) +> - [`logging.getLogger().critical()`](https://docs.python.org/3/library/logging.html#logging.Logger.critical) +> - [`logging.getLogger().error()`](https://docs.python.org/3/library/logging.html#logging.Logger.error) +> - [`logging.getLogger().warning()`](https://docs.python.org/3/library/logging.html#logging.Logger.warning) +> - [`logging.getLogger().info()`](https://docs.python.org/3/library/logging.html#logging.Logger.info) +> - [`logging.getLogger().debug()`](https://docs.python.org/3/library/logging.html#logging.Logger.debug) + +Juju automatically picks up logs from charm code that uses the Python [logging facility](https://docs.python.org/3/library/logging.html), so we can use the Juju [`debug-log` command](https://juju.is/docs/juju/juju-debug-log) to display logs for a model. Note that it shows logs from the charm code (charm container), but not the workload container. Read ["Use `juju debug-log`"](https://juju.is/docs/sdk/get-logs-from-a-kubernetes-charm#heading--use-juju-debug-log) for more information. + +Besides logs, `stderr` is also captured by Juju. So, if a charm generates a warning, it will also end up in Juju's debug log. This behaviour is consistent between K8s charms and machine charms. + +**Tips for good practice:** + +- Note that some logging is performed automatically by the Juju controller, for example when an event handler is called. Try not to replicate this behaviour in your own code. + +- Keep developer specific logging to a minimum, and use `logger.debug()` for such output. If you are debugging a problem, ensure you comment out or remove large information dumps (such as config files, etc.) from the logging once you are finished. + +- When passing messages to the logger, do not build the strings yourself. Allow the logger to do this for you as required by the specified log level. That is: + + + +```python +# Do this! +logger.info("Got some information %s", info) +# Don't do this +logger.info("Got some information {}".format(info)) +# Or this ... +logger.info(f"Got some more information {more_info}") +``` diff --git a/docs/howto/manage-relations.md b/docs/howto/manage-relations.md new file mode 100644 index 000000000..d049463bf --- /dev/null +++ b/docs/howto/manage-relations.md @@ -0,0 +1,343 @@ +(manage-relations)= +# How to manage relations + +To add integration capabilities to a charm, you’ll have to define the relation in your charm’s charmcraft.yaml file and then add relation event handlers in your charm’s `src/charm.py` file. + +> See first: [`juju` | Relation (integration)](https://juju.is/docs/juju/relation), [`juju` | Manage relations](https://juju.is/docs/juju/manage-relations), [`charmcraft` | Manage relations]() + +## Implement the feature + +### Declare the relation endpoint + +To integrate with another charm, or with itself (to communicate with other units of the same charm), declare the required and optional relations in your charm’s `charmcraft.yaml` file. + +```{caution} + + +**If you're using an existing interface:** + +Make sure to consult [the `charm-relations-interfaces` repository](https://github.com/canonical/charm-relation-interfaces) for guidance about how to implement them correctly. + +**If you're defining a new interface:** + +Make sure to add your interface to [the `charm-relations-interfaces` repository](https://github.com/canonical/charm-relation-interfaces). + + +``` + +To exchange data with other units of the same charm, define one or more `peers` endpoints including an interface name for each. Each peer relation must have an endpoint, which your charm will use to refer to the relation (as [`ops.Relation.name`](https://ops.readthedocs.io/en/latest/#ops.Relation.name)). + +```yaml +peers: + replicas: + interface: charm_gossip +``` + +To exchange data with another charm, define a `provides` or `requires` endpoint including an interface name. By convention, the interface name should be unique in the ecosystem. Each relation must have an endpoint, which your charm will use to refer to the relation (as [`ops.Relation.name`](https://ops.readthedocs.io/en/latest/#ops.Relation.name)). + +```yaml +provides: + smtp: + interface: smtp +``` + +```yaml +requires: + db: + interface: postgresql + limit: 1 +``` + +Note that implementing a cross-model relation is done in the same way as one between applications in the same model. The ops library does not distinguish between data from a different model or from the same model as the one the charm is deployed to. + +Which side of the relation is the “provider” or the “requirer” is often arbitrary, but if one side has a workload that is a server and the other a client, then the server side should be the provider. This becomes important for how Juju sets up network permissions in cross-model relations. + + +If the relation is with a subordinate charm, make sure to set the `scope` field to `container`. + +```yaml +requires: + log-forwarder: + interface: rsyslog-forwarder + scope: container +``` + +Other than this, implement a subordinate relation in the same way as any other relation. Note however that subordinate units cannot see each other’s peer data. + +> See also: [Charm taxonomy](https://juju.is/docs/sdk/charm-taxonomy#heading--subordinate-charms) + +### Add code to use a relation + +#### Using a charm library + +For most integrations, you will now want to progress with using the charm library recommended by the charm that you are integrating with. Read the documentation for the other charm on Charmhub and follow the instructions, which will typically involve adding a requirer object in your charm’s `__init__` and then observing custom events. + +In most cases, the charm library will handle observing the Juju relation events, and your charm will only need to interact with the library’s custom API. Come back to this guide when you are ready to add tests. + +> See more: [Charmhub](https://charmhub.io) + +#### Implementing your own interface + +If you are developing your own interface - most commonly for charm-specific peer data exchange, then you will need to observe the Juju relation events and add appropriate handlers. + +##### Set up a relation + +To do initial setup work when a charm is first integrated with another charm (or, in the case of a peer relation, when a charm is first deployed) your charm will need to observe the relation-created event. For example, a charm providing a database relation might need to create the database and credentials, so that the requirer charm can use the database. In the `src/charm.py` file, in the `__init__` function of your charm, set up `relation-created` event observers for the relevant relations and pair those with an event handler. + +The name of the event to observe is combined with the name of the endpoint. With an endpoint named “db”, to observe `relation-created`, our code would look like: + +```python +framework.observe(self.on.db_relation_created, self._on_db_relation_created) +``` + +Now, in the body of the charm definition, define the event handler. In this example, if we are the leader unit, then we create a database and pass the credentials to use it to the charm on the other side via the relation data: + +```python +def _on_db_relation_created(self, event: ops.RelationCreatedEvent): + if not self.unit.is_leader(): + return + credentials = self.create_database(event.app.name) + event.relation.data[event.app].update(credentials) +``` + +The event object that is passed to the handler has a `relation` property, which contains an [`ops.Relation`](https://ops.readthedocs.io/en/latest/#ops.Relation) object. Your charm uses this object to find out about the relation (such as which units are included, in the [`.units` attribute](https://ops.readthedocs.io/en/latest/#ops.Relation.units), or whether the relation is broken, in the [`.active` attribute](https://ops.readthedocs.io/en/latest/#ops.Relation.active)) and to get and set data in the relation databag. + +> See more: [`ops.RelationCreatedEvent`](https://ops.readthedocs.io/en/latest/#ops.RelationCreatedEvent) + +To do additional setup work when each unit joins the relation (both when the charms are first integrated and when additional units are added to the charm), your charm will need to observe the `relation-joined` event. In the `src/charm.py` file, in the `__init__` function of your charm, set up `relation-joined` event observers for the relevant relations and pair those with an event handler. For example: + +```python +framework.observe(self.on.smtp_relation_joined, self._on_smtp_relation_joined) +``` + +Now, in the body of the charm definition, define the event handler. In this example, a “smtp_credentials” key is set in the unit data with the ID of a secret: + +```python +def _on_smtp_relation_joined(self, event: ops.RelationJoinedEvent): + smtp_credentials_secret_id = self.create_smtp_user(event.unit.name) + event.relation.data[event.unit]["smtp_credentials"] = smtp_credentials_secret_id +``` + +> See more: [`ops.RelationJoinedEvent`](https://ops.readthedocs.io/en/latest/#ops.RelationJoinedEvent) + +##### Exchange data with other units + +To use data received through the relation, have your charm observe the `relation-changed` event. In the `src/charm.py` file, in the `__init__` function of your charm, set up `relation-changed` event observers for each of the defined relations. For example: + +```python +framework.observe(self.on.replicas_relation_changed, self._update_configuration) +``` + +> See more: [[`ops.RelationChangedEvent`](https://ops.readthedocs.io/en/latest/#ops.RelationChangedEvent)](https://discourse.charmhub.io/t/relation-name-relation-changed-event/6475), [`juju` | Relation (integration)](https://juju.is/docs/juju/relation#heading--permissions-around-relation-databags) + +Most of the time, you should use the same holistic handler as when receiving other data, such as `secret-changed` and `config-changed`. To access the relation(s) in your holistic handler, use the [`ops.Model.get_relation`](https://ops.readthedocs.io/en/latest/#ops.Model.get_relation) method or [`ops.Model.relations`](https://ops.readthedocs.io/en/latest/#ops.Model.relations) attribute. + +> See also: {ref}`holistic-vs-delta-charms` + +If your change will have at most one relation on the endpoint, to get the `Relation` object use `Model.get_relation`; for example: + +```python +rel = self.model.get_relation("db") +if not rel: + # Handle the case where the relation does not yet exist. +``` + +If your charm may have multiple relations on the endpoint, to get the relation objects use `Model.relations` rather than `Model.get_relation` with the relation ID; for example: + +```python +for rel in self.model.relations.get('smtp', ()): + # Do something with the relation object. +``` + +Once your charm has the relation object, it can be used in exactly the same way as when received from an event. + +Now, in the body of the charm definition, define the holistic event handler. In this example, we check if the relation exists yet, and for a provided secret using the ID provided in the relation data, and if we have both of those then we push that into a workload configuration: + +```python +def _update_configuration(self, _: ops.Eventbase): + # This handles secret-changed and relation-changed. + db_relation = self.model.get_relation('db') + if not db_relation: + # We’re not integrated with the database charm yet. + return + secret_id = db_relation.data[self.model.app]['credentials'] + if not secret_id: + # The credentials haven’t been added to the relation by the remote app yet. + return + secret_contents = self.model.get_secret(id=secret_id).get_contents(refresh=True) + self.push_configuration( + username=secret['username'], + password=secret['password'], + ) +``` + +##### Exchange data across the various relations + +To add data to the relation databag, use the [`.data` attribute](https://ops.readthedocs.io/en/latest/#ops.Relation.data) much as you would a dictionary, after selecting whether to write to the app databag (leaders only) or unit databag. For example, to copy a value from the charm config to the relation data: + +```python +def _on_config_changed(self, event: ops.ConfigChangedEvent): + if relation := self.model.get_relation('ingress'): + relation.data[self.app]["domain"] = self.model.config["domain"] +``` + +To read data from the relation databag, again use the `.data` attribute, selecting the appropriate databag, and then using it as if it were a regular dictionary. + +The charm can inspect the contents of the remote unit databags: + +```python +def _on_database_relation_changed(self, event: ops.RelationChangedEvent): + remote_units_databags = { + event.relation.data[unit] for unit in event.relation.units if unit.app is not self.app + } +``` + +Or the peer unit databags: + +```python +def _on_database_relation_changed(self, e: ops.RelationChangedEvent): + peer_units_databags = { + event.relation.data[unit] for unit in event.relation.units if unit.app is self.app + } +``` + +Or the remote leader databag: + +```python +def _on_database_relation_changed(self, event: ops.RelationChangedEvent): + remote_app_databag = event.relation.data[relation.app] +``` + +Or the local application databag: + +```python +def _on_database_relation_changed(self, event: ops.RelationChangedEvent): + local_app_databag = event.relation.data[self.app] +``` + +Or the local unit databag: + +```python +def _on_database_relation_changed(self, event: ops.RelationChangedEvent): + local_unit_databag = event.relation.data[self.unit] +``` + +If the charm does not have permission to do an operation (e.g. because it is not the leader unit), an exception will be raised. + +##### Clean up when a relation is removed + +To do clean-up work when a unit in the relation is removed (for example, removing per-unit credentials), have your charm observe the `relation-departed` event. In the `src/charm.py` file, in the `__init__` function of your charm, set up `relation-departed` event observers for the relevant relations and pair those with an event handler. For example: + +```python +framework.observe(self.on.smtp_relation_departed, self._on_smtp_relation_departed) +``` + +Now, in the body of the charm definition, define the event handler. For example: + +```python +def _on_smtp_relation_departed(self, event: ops.RelationDepartedEvent): + if self.unit != event.departing_unit: + self.remove_smtp_user(event.unit.name) +``` + +> See more: [ops.RelationDepartedEvent](https://ops.readthedocs.io/en/latest/#ops.RelationDepartedEvent) + +To clean up after a relation is entirely removed, have your charm observe the `relation-broken` event. In the `src/charm.py` file, in the `__init__` function of your charm, set up `relation-broken` events for the relevant relations and pair those with an event handler. For example: + +```python +framework.observe(self.on.db_relation_broken, self._on_db_relation_broken) +``` + +Now, in the body of the charm definition, define the event handler. For example: + +```python +def _on_db_relation_broken(self, event: ops.RelationBrokenEvent): + if not self.is_leader(): + return + self.drop_database(event.app.name) +``` + +> See more: [ops.RelationBrokenEvent](https://ops.readthedocs.io/en/latest/#ops.RelationBrokenEvent) + + +## Test the feature + +### Write unit tests + +To write unit tests covering your charm’s behaviour when working with relations, in your `unit/test_charm.py` file, create a `Harness` object and use it to simulate adding and removing relations, or the remote app providing data. For example: + +```python +@pytest.fixture() +def harness(): + harness = testing.Harness(MyCharm) + yield harness + harness.cleanup() + +def test_new_smtp_relation(harness): + # Before the test begins, we have integrated a remote app + # with this charm, so we call add_relation() before begin(). + relation_id = harness.add_relation('smtp', 'consumer_app') + harness.begin() + # For the test, we simulate a unit joining the relation. + harness.add_relation_unit() + assert 'smtp_credentials’ in harness.get_relation_data(relation_id, 'consumer_app/0' ) + +def test_db_relation_broken(harness): + relation_id = harness.add_relation('db', 'postgresql') + harness.begin() + harness.remove_relation(relation_id) + assert harness.charm.get_db() is None + +def test_receive_db_credentials(harness): + relation_id = harness.add_relation('db', 'postgresql') + harness.begin() + harness.update_relation_data(relation_id, harness.charm.app, {'credentials-id': 'secret:xxx'}) + assert harness.charm.db_tables_created() +``` + +> See more: [ops.testing.Harness](https://ops.readthedocs.io/en/latest/harness.html#ops.testing.Harness) + +### Write scenario tests + +For each relation event that your charm observes, write at least one Scenario test. Create a `Relation` object that defines the relation, include that in the input state, run the relation event, and assert that the output state is what you’d expect. For example: + +```python +ctx = scenario.Context(MyCharm) +relation = scenario.Relation(id=1, endpoint='smtp', remote_units_data={1: {}}) +state_in = scenario.State(relations=[relation]) +state_out = context.run(relation.joined_event(remote_unit_id=1), state=state_in) +assert 'smtp_credentials' in state_out.relations[0].remote_units_data[1] +``` + +> See more: [Scenario Relations](https://github.com/canonical/ops-scenario/#relations) + +### Write integration tests + +Write an integration test that verifies that your charm behaves as expected in a real Juju environment. Other than when testing peer relations, as well as deploying your own charm, the test needs to deploy a second application, either the real charm or a facade that provides enough functionality to test against. + +The pytest-operator plugin provides methods to deploy multiple charms. For example: + +```python +# This assumes that your integration tests already include the standard +# build and deploy test that the charmcraft profile provides. + +@pytest.mark.abort_on_fail +async def test_active_when_deploy_db_facade(ops_test: OpsTest): + await ops_test.model.deploy('facade') + await ops_test.model.integrate(APP_NAME + ':postgresql', 'facade:provide-postgresql') + + present_facade('postgresql', model=ops_test.model_name, + app_data={ + 'credentials': 'secret:abc', + }) + + await ops_test.model.wait_for_idle( + apps=[APP_NAME], + status='active', + timeout=600, + ) +``` + +> See more: [`pytest-operator`](https://pypi.org/project/pytest-operator/) + + diff --git a/docs/howto/manage-resources.md b/docs/howto/manage-resources.md new file mode 100644 index 000000000..d23143f40 --- /dev/null +++ b/docs/howto/manage-resources.md @@ -0,0 +1,70 @@ +(manage-resources)= +# How to manage resources + +> See also: [`juju` | Resource (charm)](https://juju.is/docs/juju/charm-resource), [`juju` | Manage resources](https://juju.is/docs/juju/manage-charm-resources), [`charmcraft` | Manage resources]() + +## Implement the feature + + + +In your charm's `src/charm.py` file, use Ops to fetch the path to the resource and then manipulate it as needed. + +For example, suppose your `charmcraft.yaml` file contains this simple resource definition: + +```yaml +resources: + my-resource: + type: file + filename: somefile.txt + description: test resource +``` + +In your charm's `src/charm.py` you can now use [`Model.resources.fetch()`](https://ops.readthedocs.io/en/latest/#ops.Resources.fetch) to get the path to the resource, then manipulate it as needed. For example: + +```python +# ... +import logging +import ops +# ... +logger = logging.getLogger(__name__) + +def _on_config_changed(self, event): + # Get the path to the file resource named 'my-resource' + try: + resource_path = self.model.resources.fetch("my-resource") + except ops.ModelError as e: + self.unit.status = ops.BlockedStatus( + "Something went wrong when claiming resource 'my-resource; " + "run `juju debug-log` for more info'" + ) + # might actually be worth it to just reraise this exception and let the charm error out; + # depends on whether we can recover from this. + logger.error(e) + return + except NameError as e: + self.unit.status = ops.BlockedStatus( + "Resource 'my-resource' not found; " + "did you forget to declare it in charmcraft.yaml?" + ) + logger.error(e) + return + + # Open the file and read it + with open(resource_path, "r") as f: + content = f.read() + # do something +``` + +The [`fetch()`](https://ops.readthedocs.io/en/latest/#ops.Resources.fetch) method will raise a [`NameError`](https://docs.python.org/3/library/exceptions.html#NameError) if the resource does not exist, and returns a Python [`Path`](https://docs.python.org/3/library/pathlib.html#pathlib.Path) object to the resource if it does. + + +Note: During development, it may be useful to specify the resource at deploy time to facilitate faster testing without the need to publish a new charm/resource in between minor fixes. In the below snippet, we create a simple file with some text content, and pass it to the Juju controller to use in place of any published `my-resource` resource: + +```text +echo "TEST" > /tmp/somefile.txt +charmcraft pack +juju deploy ./my-charm.charm --resource my-resource=/tmp/somefile.txt +``` + diff --git a/docs/howto/manage-secrets.md b/docs/howto/manage-secrets.md new file mode 100644 index 000000000..3bfea3532 --- /dev/null +++ b/docs/howto/manage-secrets.md @@ -0,0 +1,377 @@ +(manage-secrets)= +# How to manage secrets + +> See first: [`juju` | Secret](https://juju.is/docs/juju/secret), [`juju` | Manage secrets](https://juju.is/docs/juju/manage-secrets), [`charmcraft` | Manage secrets]() + +> Added in `ops 2.0.0`, `juju 3.0.2` + +This document shows how to use secrets in a charm -- both when the charm is the secret owner as well as when it is merely an observer. + +## Secret owner charm + +> By its nature, the content in this section only applies to *charm* secrets. + +### Add and grant access to a secret + +Before secrets, the owner charm might have looked as below: + +```python +class MyDatabaseCharm(ops.CharmBase): + def __init__(self, *args, **kwargs): + ... # other setup + self.framework.observe(self.on.database_relation_joined, + self._on_database_relation_joined) + + ... # other methods and event handlers + + def _on_database_relation_joined(self, event: ops.RelationJoinedEvent): + event.relation.data[self.app]['username'] = 'admin' + event.relation.data[self.app]['password'] = 'admin' # don't do this at home +``` + +With secrets, this can be rewritten as: + +```python +class MyDatabaseCharm(ops.CharmBase): + def __init__(self, *args, **kwargs): + ... # other setup + self.framework.observe(self.on.database_relation_joined, + self._on_database_relation_joined) + + ... # other methods and event handlers + + def _on_database_relation_joined(self, event: ops.RelationJoinedEvent): + content = { + 'username': 'admin', + 'password': 'admin', + } + secret = self.app.add_secret(content) + secret.grant(event.relation) + event.relation.data[self.app]['secret-id'] = secret.id +``` + +Note that: +- We call `add_secret` on `self.app` (the application). That is because we want the secret to be owned by this application, not by this unit. If we wanted to create a secret owned by the unit, we'd call `self.unit.add_secret` instead. +- The only data shared in plain text is the secret ID (a locator URI). The secret ID can be publicly shared. Juju will ensure that only remote apps/units to which the secret has explicitly been granted by the owner will be able to fetch the actual secret payload from that ID. +- The secret needs to be granted to a remote entity (app or unit), and that always goes via a relation instance. By passing a relation to `grant` (in this case the event's relation), we are explicitly declaring the scope of the secret -- its lifetime will be bound to that of this relation instance. + +> See more: [`ops.Application.add_secret()`](https://ops.readthedocs.io/en/latest/#ops.Application.add_secret), + + + +### Create a new secret revision + + +To create a new secret revision, the owner charm must call `secret.set_content()` and pass in the new payload: + +```python +class MyDatabaseCharm(ops.CharmBase): + + ... # as before + + def _rotate_webserver_secret(self, secret): + content = secret.get_content() + secret.set_content({ + 'username': content['username'], # keep the same username + 'password': _generate_new_secure_password(), # something stronger than 'admin' + }) +``` + +This will inform Juju that a new revision is available, and Juju will inform all observers tracking older revisions that a new one is available, by means of a `secret-changed` hook. + + +### Change the rotation policy or the expiration date of a secret + +Typically you want to rotate a secret periodically to contain the damage from a leak, or to avoid giving hackers too much time to break the encryption. + + +A charm can configure a secret, at creation time, to have one or both of: + +- A rotation policy (weekly, monthly, daily, and so on). +- An expiration date (for example, in two months from now). + +Here is what the code would look like: + +```python +class MyDatabaseCharm(ops.CharmBase): + def __init__(self, *args, **kwargs): + ... # other setup + self.framework.observe(self.on.secret_rotate, + self._on_secret_rotate) + + ... # as before + + def _on_database_relation_joined(self, event: ops.RelationJoinedEvent): + content = { + 'username': 'admin', + 'password': 'admin', + } + secret = self.app.add_secret(content, + label='secret-for-webserver-app', + rotate=SecretRotate.DAILY) + + def _on_secret_rotate(self, event: ops.SecretRotateEvent): + # this will be called once per day. + if event.secret.label == 'secret-for-webserver-app': + self._rotate_webserver_secret(event.secret) +``` + +Or, for secret expiration: + +```python +class MyDatabaseCharm(ops.CharmBase): + def __init__(self, *args, **kwargs): + ... # other setup + self.framework.observe(self.on.secret_expired, + self._on_secret_expired) + + ... # as before + + def _on_database_relation_joined(self, event: ops.RelationJoinedEvent): + content = { + 'username': 'admin', + 'password': 'admin', + } + secret = self.app.add_secret(content, + label='secret-for-webserver-app', + expire=datetime.timedelta(days=42)) # this can also be an absolute datetime + + def _on_secret_expired(self, event: ops.SecretExpiredEvent): + # this will be called only once, 42 days after the relation-joined event. + if event.secret.label == 'secret-for-webserver-app': + self._rotate_webserver_secret(event.secret) +``` + + + + +### Remove a secret + + +To remove a secret (effectively destroying it for good), the owner needs to call `secret.remove_all_revisions`. Regardless of the logic leading to the decision of when to remove a secret, the code will look like some variation of the following: + +```python +class MyDatabaseCharm(ops.CharmBase): + ... + + # called from an event handler + def _remove_webserver_secret(self): + secret = self.model.get_secret(label='secret-for-webserver-app') + secret.remove_all_revisions() +``` + +After this is called, the observer charm will get a `ModelError` whenever it attempts to get the secret. In general, the presumption is that the observer charm will take the absence of the relation as indication that the secret is gone as well, and so will not attempt to get it. + + +### Remove a single secret revision + +Removing a single secret revision is a more common (and less drastic!) operation than removing all revisions. + +Typically, the owner will remove a secret revision when it receives a `secret-remove` event -- that is, when that specific revision is no longer tracked by any observer. If a secret owner did remove a revision while it was still being tracked by observers, they would get a `ModelError` when they tried to get the secret. + +A typical implementation of the `secret-remove` event would look like: + +```python +class MyDatabaseCharm(ops.CharmBase): + + ... # as before + + def __init__(self, *args, **kwargs): + ... # other setup + self.framework.observe(self.on.secret_remove, + self._on_secret_remove) + + def _on_secret_remove(self, event: ops.SecretRemoveEvent): + # all observers are done with this revision, remove it + event.secret.remove_revision(event.revision) +``` + + +### Revoke a secret + +For whatever reason, the owner of a secret can decide to revoke access to the secret to a remote entity. That is done by calling `secret.revoke`, and is the inverse of `secret.grant`. + +An example of usage might look like: +```python +class MyDatabaseCharm(ops.CharmBase): + + ... # as before + + # called from an event handler + def _revoke_webserver_secret_access(self, relation): + secret = self.model.get_secret(label='secret-for-webserver-app') + secret.revoke(relation) +``` + +Just like when the owner granted the secret, we need to pass a relation to the `revoke` call, making it clear what scope this action is to be applied to. + + +## Secret observer charm + +> This applies to both charm and user secrets, though for user secrets the story starts with the charm defining a configuration option of type `secret`, and the secret is not acquired through relation data but rather by the configuration option being set to the secret's URI. +> +> A secret owner charm is also an observer of the secret, so this applies to it too. + +### Start tracking the latest secret revision + +Before secrets, the code in the secret observer charm may have looked something like this: + +```python +class MyWebserverCharm(ops.CharmBase): + def __init__(self, *args, **kwargs): + ... # other setup + self.framework.observe(self.on.database_relation_changed, + self._on_database_relation_changed) + + ... # other methods and event handlers + + def _on_database_relation_changed(self, event: ops.RelationChangedEvent): + username = event.relation.data[event.app]['username'] + password = event.relation.data[event.app]['password'] + self._configure_db_credentials(username, password) +``` + +With secrets, the code would become: + +```python +class MyWebserverCharm(ops.CharmBase): + def __init__(self, *args, **kwargs): + ... # other setup + self.framework.observe(self.on.database_relation_changed, + self._on_database_relation_changed) + + ... # other methods and event handlers + + def _on_database_relation_changed(self, event: ops.RelationChangedEvent): + secret_id = event.relation.data[event.app]['secret-id'] + secret = self.model.get_secret(id=secret_id) + content = secret.get_content() + self._configure_db_credentials(content['username'], content['password']) +``` + +Note that: +- The observer charm gets a secret via the model (not its app/unit). Because it's the owner who decides who the secret is granted to, the ownership of a secret is not an observer concern. The observer code can rightfully assume that, so long as a secret ID is shared with it, the owner has taken care to grant and scope the secret in such a way that the observer has the rights to inspect its contents. +- The charm first gets the secret object from the model, then gets the secret's content (a dict) and accesses individual attributes via the dict's items. + + +> See more: [`ops.Secret.get_content()`](https://ops.readthedocs.io/en/latest/#ops.Secret.get_content) + +### Label the secrets you're observing + + +Sometimes a charm will observe multiple secrets. In the `secret-changed` event handler above, you might ask yourself: How do I know which secret has changed? +The answer lies with **secret labels**: a label is a charm-local name that you can assign to a secret. Let's go through the following code: + +```python +class MyWebserverCharm(ops.CharmBase): + + ... # as before + + def _on_database_relation_changed(self, event: ops.RelationChangedEvent): + secret_id = event.relation.data[event.app]['secret-id'] + secret = self.model.get_secret(id=secret_id, label='database-secret') + content = secret.get_content() + self._configure_db_credentials(content['username'], content['password']) + + def _on_secret_changed(self, event: ops.SecretChangedEvent): + if event.secret.label == 'database-secret': + content = event.secret.get_content(refresh=True) + self._configure_db_credentials(content['username'], content['password']) + elif event.secret.label == 'my-other-secret': + self._handle_other_secret_changed(event.secret) + else: + pass # ignore other labels (or log a warning) +``` + +As shown above, when the web server charm calls `get_secret` it can specify an observer-specific label for that secret; Juju will attach this label to the secret at that point. Normally `get_secret` is called for the first time in a relation-changed event; the label is applied then, and subsequently used in a secret-changed event. + +Labels are unique to the charm (the observer in this case): if you attempt to attach a label to two different secrets from the same application (whether it's the on the observer side or the owner side) and give them the same label, the framework will raise a `ModelError`. + +Whenever a charm receives an event concerning a secret for which it has set a label, the label will be present on the secret object exposed by the framework. + +The owner of the secret can do the same. When a secret is added, you can specify a label for the newly-created secret: + +```python +class MyDatabaseCharm(ops.CharmBase): + + ... # as before + + def _on_database_relation_joined(self, event: ops.RelationJoinedEvent): + content = { + 'username': 'admin', + 'password': 'admin', + } + secret = self.app.add_secret(content, label='secret-for-webserver-app') + secret.grant(event.relation) + event.relation.data[event.unit]['secret-id'] = secret.id +``` + +If a secret has been labelled in this way, the charm can retrieve the secret object at any time by calling `get_secret` with the "label" argument. This way, a charm can perform any secret management operation even if all it knows is the label. The secret ID is normally only used to exchange a reference to the secret *between* applications. Within a single application, all you need is the secret label. + +So, having labelled the secret on creation, the database charm could add a new revision as follows: + +```python + def _rotate_webserver_secret(self): + secret = self.model.get_secret(label='secret-for-webserver-app') + secret.set_content(...) # pass a new revision payload, as before +``` + +> See more: [`ops.Model.get_secret()`](https://ops.readthedocs.io/en/latest/#ops.Model.get_secret) + +#### When to use labels + +When should you use labels? A label is basically the secret's *name* (local to the charm), so whenever a charm has, or is observing, multiple secrets you should label them. This allows you to distinguish between secrets, for example, in the `SecretChangedEvent` shown above. + +Most charms that use secrets have a fixed number of secrets each with a specific meaning, so the charm author should give them meaningful labels like `database-credential`, `tls-cert`, and so on. Think of these as "pets" with names. + +In rare cases, however, a charm will have a set of secrets all with the same meaning: for example, a set of TLS certificates that are all equally valid. In this case it doesn't make sense to label them -- think of them as "cattle". To distinguish between secrets of this kind, you can use the [`Secret.unique_identifier`](https://ops.readthedocs.io/en/latest/#ops.Secret.unique_identifier) property, added in ops 2.6.0. + +Note that [`Secret.id`](https://ops.readthedocs.io/en/latest/#ops.Secret.id), despite the name, is not really a unique ID, but a locator URI. We call this the "secret ID" throughout Juju and in the original secrets specification -- it probably should have been called "uri", but the name stuck. + + + + +### Peek at a new secret revision + +Sometimes, before reconfiguring to use a new credential revision, the observer charm may want to peek at its contents (for example, to ensure that they are valid). Use `peek_content` for that: + +```python + def _on_secret_changed(self, event: ops.SecretChangedEvent): + content = event.secret.peek_content() + if not self._valid_password(content.get('password')): + logger.warning('Invalid credentials! Not updating to new revision.') + return + content = event.secret.get_content(refresh=True) + ... +``` + +> See more: [`ops.Secret.peek_content()`](https://ops.readthedocs.io/en/latest/#ops.Secret.peek_content) + + +### Start tracking a different secret revision + + +To update to a new revision, the web server charm will typically subscribe to the `secret-changed` event and call `get_content` with the "refresh" argument set (refresh asks Juju to start tracking the latest revision for this observer). + +```python +class MyWebserverCharm(ops.CharmBase): + def __init__(self, *args, **kwargs): + ... # other setup + self.framework.observe(self.on.secret_changed, + self._on_secret_changed) + + ... # as before + + def _on_secret_changed(self, event: ops.SecretChangedEvent): + content = event.secret.get_content(refresh=True) + self._configure_db_credentials(content['username'], content['password']) +``` + + +> See more: [`ops.Secret.get_content()`](https://ops.readthedocs.io/en/latest/#ops.Secret.get_content) + + + +
+ +**Contributors:** @ppasotti, @tmihoc, @tony-meyer, @wallyworld diff --git a/docs/howto/manage-storage.md b/docs/howto/manage-storage.md new file mode 100644 index 000000000..19ddc8933 --- /dev/null +++ b/docs/howto/manage-storage.md @@ -0,0 +1,247 @@ +(manage-storage)= +# How to manage storage + +> See first: [`juju` | Storage](https://juju.is/docs/juju/storage), [`juju` | Manage storage](https://juju.is/docs/juju/manage-storage), [`charmcraft | Manage storage]() + +## Implement the feature + +### Declare the storage + +To define the storage that can be provided to the charm, define a `storage` section in `charmcraft.yaml` that lists the storage volumes and information about each storage. For example, for a transient filesystem storage mounted to `/cache/` that is at least 1GB in size: + +```yaml +storage: + local-cache: + type: filesystem + description: Somewhere to cache files locally. + location: /cache/ + minimum-size: 1G + properties: + - transient +``` + +For Kubernetes charms, you also need to define where on the workload container the volume will be mounted. For example, to mount a similar cache filesystem in `/var/cache/`: + +```yaml +storage: + local-cache: + type: filesystem + description: Somewhere to cache files locally. + # The location is not required here, because it defines the location on + # the charm container, not the workload container. + minimum-size: 1G + properties: + - transient + +containers: + web-service: + resource: app-image + mounts: + - storage: local-cache + location: /var/cache +``` + + + +### Observe the `storage-attached` event and define an event handler + +In the `src/charm.py` file, in the `__init__` function of your charm, set up an observer for the `storage-attached` event associated with your storage and pair that with an event handler, typically a holistic one. For example: + +``` +self.framework.observe(self.on.cache_storage_attached, self._update_configuration) +``` + +> See more: [`ops.StorageAttachedEvent`](https://ops.readthedocs.io/en/latest/#ops.StorageAttachedEvent), [Juju SDK | Holistic vs delta charms](https://juju.is/docs/sdk/holistic-vs-delta-charms) + +Storage volumes will be automatically mounted into the charm container at either the path specified in the `location` field in the metadata, or the default location `/var/lib/juju/storage/`. However, your charm code should not hard-code the location, and should instead use the `.location` property of the storage object. + +Now, in the body of the charm definition, define the event handler, or adjust an existing holistic one. For example, to provide the location of the attached storage to the workload configuration: + +``` +def _update_configuration(self, event: ops.EventBase): + """Update the workload configuration.""" + cache = self.model.storages["cache"] + if cache.location is None: + # This must be one of the other events. Return and wait for the storage-attached event. + logger.info("Storage is not yet ready.") + return + try: + self.push_configuration(cache_dir=cache.location) + except ops.pebble.ConnectionError: + # Pebble isn't ready yet. Return and wait for the pebble-ready event. + logger.info("Pebble is not yet ready.") + return +``` + +> Examples: [ZooKeeper ensuring that permission and ownership is correct](https://github.com/canonical/zookeeper-operator/blob/106f9c2cd9408a172b0e93f741d8c9f860c4c38e/src/charm.py#L247), [Kafka configuring additional storage](https://github.com/canonical/kafka-k8s-operator/blob/25cc5dd87bc2246c38fc511ac9c52f35f75f6513/src/charm.py#L298) + +### Observe the detaching event and define an event handler + +In the `src/charm.py` file, in the `__init__` function of your charm, set up an observer for the detaching event associated with your storage and pair that with an event handler. For example: + +``` +self.framework.observe(self.on.cache_storage_detaching, self._on_storage_detaching) +``` + +> See more: [`ops.StorageDetachingEvent`](https://ops.readthedocs.io/en/latest/#ops.StorageDetachingEvent) + +Now, in the body of the charm definition, define the event handler, or adjust an existing holistic one. For example, to warn users that data won't be cached: + +``` +def _on_storage_detaching(self, event: ops.StorageDetachingEvent): + """Handle the storage being detached.""" + self.unit.status = ops.ActiveStatus("Caching disabled; provide storage to boost performance) +``` + +> Examples: [MySQL handling cluster management](https://github.com/canonical/mysql-k8s-operator/blob/4c575b478b7ae2a28b09dde9cade2d3370dd4db6/src/charm.py#L823), [MongoDB updating the set before storage is removed](https://github.com/canonical/mongodb-operator/blob/b33d036173f47c68823e08a9f03189dc534d38dc/src/charm.py#L596) + +### Request additional storage + +```{note} + +Juju only supports adding multiple instances of the same storage volume on machine charms. Kubernetes charms may only have a single instance of each volume. + +``` + +If the charm needs additional units of a storage, it can request that with the `storages.request` +method. The storage must be defined in the metadata as allowing multiple, for +example: + +```yaml +storage: + scratch: + type: filesystem + location: /scratch + multiple: 1-10 +``` + +For example, if the charm needs to request two additional units of this storage: + +```python +self.model.storages.request("scratch", 2) +``` + +The storage will not be available immediately after that call - the charm should +observe the `storage-attached` event and handle any remaining setup once Juju +has attached the new storage. + +## Test the feature + +> See first: {ref}`get-started-with-charm-testing>` + +You'll want to add three levels of tests: + +### Write unit tests + +> See first: {ref}`write-unit-tests-for-a-charm` + +When using Harness for unit tests, use the `add_storage()` method to simulate Juju adding storage to the charm. You can either have the method also simulate attaching the storage, or do that explicitly with the `attach_storage()` method. In this example, we verify that the charm responds as expected to storage attached and detaching events: + +```python +@pytest.fixture() +def harness(): + yield ops.testing.Harness(MyCharm) + harness.cleanup() + + +def test_storage_attached(harness): + # Add one instance of the expected storage to the charm. This is before `.begin()` is called, + # so will not trigger any events. + storage_id = harness.add_storage("cache", 1) + harness.begin() + # Simulate Juju attaching the storage, which will trigger a storage-attached event on the charm. + harness.attach_storage(storage_id) + # Assert that it was handled correctly. + assert ... + + +def test_storage_detaching(harness): + storage_id = harness.add_storage("cache", 1, attach=True) + harness.begin() + # Simulate the harness being detached (.remove_storage() would simulate it being removed + # entirely). + harness.remove_storage(storage_id) + # Asser that it was handled correctly. + assert ... +``` + +> See more: [`ops.testing.Harness.add_storage`](https://ops.readthedocs.io/en/latest/harness.html#ops.testing.Harness.add_storage), [`ops.testing.Harness.attach_storage`](https://ops.readthedocs.io/en/latest/harness.html#ops.testing.Harness.attach_storage), [`ops.testing.Harness.detach_storage`](https://ops.readthedocs.io/en/latest/harness.html#ops.testing.Harness.detach_storage), [`ops.testing.harness.remove_storage`](https://ops.readthedocs.io/en/latest/harness.html#ops.testing.Harness.remove_storage) + +### Write scenario tests + +> See first: {ref}`write-scenario-tests-for-a-charm` + +When using Scenario for unit tests, to verify that the charm state is as expected after storage changes, use the `run` method of the Scenario `Context` object. For example, to provide the charm with mock storage: + +```python +# Some charm with a 'foo' filesystem-type storage defined in its metadata: +ctx = scenario.Context(MyCharm) +storage = scenario.Storage("foo") + +# Set up storage with some content: +(storage.get_filesystem(ctx) / "myfile.txt").write_text("helloworld") + +with ctx.manager("update-status", scenario.State(storage=[storage])) as mgr: + foo = mgr.charm.model.storages["foo"][0] + loc = foo.location + path = loc / "myfile.txt" + assert path.exists() + assert path.read_text() == "helloworld" + + myfile = loc / "path.py" + myfile.write_text("helloworlds") + +# Verify that the contents are as expected afterwards. +assert ( + storage.get_filesystem(ctx) / "path.py" +).read_text() == "helloworlds" +``` + +If a charm requests adding more storage instances while handling some event, you +can inspect that from the `Context.requested_storage` API. + +```python +ctx = scenario.Context(MyCharm) +ctx.run('some-event-that-will-request-more-storage', scenario.State()) + +# The charm has requested two 'foo' storage volumes to be provisioned: +assert ctx.requested_storages['foo'] == 2 +``` + +Requesting storage volumes has no other consequence in Scenario. In real life, +this request will trigger Juju to provision the storage and execute the charm +again with foo-storage-attached. So a natural follow-up Scenario test suite for +this case would be: + +``` +ctx = scenario.Context(MyCharm) +foo_0 = scenario.Storage('foo') +# The charm is notified that one of the storage volumes it has requested is ready: +ctx.run(foo_0.attached_event, State(storage=[foo_0])) + +foo_1 = scenario.Storage('foo') +# The charm is notified that the other storage is also ready: +ctx.run(foo_1.attached_event, State(storage=[foo_0, foo_1])) +``` + +> See more: [Scenario storage testing](https://github.com/canonical/ops-scenario/#storage) + + +### Write integration tests + +> See first: {ref}`write-integration-tests-for-a-charm` + +To verify that adding and removing storage works correctly against a real Juju instance, write an integration test with `pytest_operator`. For example: + +```python +# This assumes there is a previous test that handles building and deploying. +async def test_storage_attaching(ops_test): + # Add a 1GB "cache" storage: + await ops_test.model.applications[APP_NAME].units[0].add_storage("cache", size=1024*1024) + + await ops_test.model.wait_for_idle( + apps=[APP_NAME], status="active", timeout=600 + ) + + # Assert that the storage is being used appropriately. +``` diff --git a/docs/howto/manage-the-charm-version.md b/docs/howto/manage-the-charm-version.md new file mode 100644 index 000000000..3bf6df17a --- /dev/null +++ b/docs/howto/manage-the-charm-version.md @@ -0,0 +1,87 @@ +(manage-the-charm-version)= +# How to manage the charm version + +## Implement the feature + +Charms can specify a version of the charm itself, so that a Juju admin can track +the installed version of the charm back to the source tree that it was built +from. + +To set the version, in the root directory of your charm (at the same level as +the `charmcraft.yaml` file) add a file called `version` (no extension). The +content of the file is the version string, which is typically a +major.minor.patch style version that's manually updated, or a version control +hash identifier. + +For example, using the hash of the latest HEAD commit as the version: + +```shell +$ git rev-parse HEAD > version +$ ls +lib src tox.ini charmcraft.yaml LICENSE requirements.txt tests version +$ cat version +0522e1fd009dac78adb3d0652d91a1e8ff7982ae +``` + +```{note} + +Normally, your publishing workflow would take care of updating this file, so +that it will automatically match the revision that you're publishing. Generally, +using a version control revision is the best choice, as it unambiguously +identifies the code that was used to build the charm. + +``` + +Juju admins using your charm can find this information with `juju status` in the +YAML or JSON formats in the `applications..charm-version` field. If +there is no version, the key will not be present in the status output. + +Note that this is distinct from the charm **revision**, which is set when +uploading a charm to CharmHub (or when deploying/refreshing for local charms). + + +> Examples: [`container-log-archive-charm` sets `version` to a version control hash](https://git.launchpad.net/container-log-archive-charm/tree/) + +## Test the feature + +> See first: {ref}`get-started-with-charm-testing` + +Since the version isn't set by the charm code itself, you'll want to test that +the version is correctly set with an integration test, and don't need to write +a unit test. + +> See first: {ref}`write-integration-tests-for-a-charm` + +To verify that setting the charm version works correctly in an integration test, +in your `tests/integration/test_charm.py` file, add a new test after the +`test_build_and_deploy` one that `charmcraft init` provides. In this test, get +the status of the model, and check the `charm_version` attribute of the unit. +For example: + +```python +# `charmcraft init` will provide this test for you. +async def test_build_and_deploy(ops_test: OpsTest): + # Build and deploy charm from local source folder + charm = await ops_test.build_charm(".") + + # Deploy the charm and wait for active/idle status + await asyncio.gather( + ops_test.model.deploy(charm, application_name=APP_NAME), + ops_test.model.wait_for_idle( + apps=[APP_NAME], status="active", raise_on_blocked=True, timeout=1000 + ), + ) + +async def test_charm_version_is_set(ops_test: OpsTest): + # Verify that the charm version has been set. + status = await ops_test.model.get_status() + version = status.applications[APP_NAME].charm_version + expected_version = subprocess.check_output(["git", "rev-parse", "HEAD"]).decode("utf8") + assert version == expected_version +``` + + + +> Examples: [synapse checking that the unit's workload version matches the one reported by the server](https://github.com/canonical/synapse-operator/blob/778bcd414644c922373d542a304be14866835516/tests/integration/test_charm.py#L139) diff --git a/docs/howto/manage-the-workload-version.md b/docs/howto/manage-the-workload-version.md new file mode 100644 index 000000000..988f5d109 --- /dev/null +++ b/docs/howto/manage-the-workload-version.md @@ -0,0 +1,154 @@ +(how-to-set-the-workload-version)= +# How to set the workload version + +## Implement the feature + +Applications modelled by charms have their own version; each application +will have its own versioning scheme, and its own way of accessing that +information. To make things easier for Juju admins, the charm should expose the +workload version through Juju - it will be visible in `juju status` (in the +default tabular view, in the application table, in the "Version" column; in the +JSON or YAML format, under `applications..version`). + +```{note} + +If the charm has not set the workload version, then the field will not be +present in JSON or YAML format, and if the version string is too long or +contains particular characters then it will not be displayed in the tabular +format. + +``` + +For Kubernetes charms, the workload is typically started in the +`-pebble-ready` event, and the version can be retrieved and passed +to Juju at that point. If the workload cannot immediately provide a version +string, then your charm will need to do this in a later event instead. + +For machine charms, the workload should be available in the `start` event, so +you can retrieve the version from it and pass it to Juju in a `start` event +handler. In this case, if you don't already have a `start` handler, in the +`src/charm.py` file, in the `__init__` function of your charm, set up an +observer for the `start` event and pair that with an event handler. For example: + +```python +self.framework.observe(self.on.start, self._on_start) +``` + +> See more: [`ops.StartEvent`](https://ops.readthedocs.io/en/latest/#ops.StartEvent) + +Now, in the body of the charm definition, define the event handler. Typically, +the workload version is retrieved from the workload itself, with a subprocess +(machine charms) or Pebble exec (Kubernetes charms) call or HTTP request. For +example: + +```python +def _on_start(self, event: ops.StartEvent): + # The workload exposes the version via HTTP at /version + version = requests.get("http://localhost:8000/version").text + self.unit.set_workload_version(version) +``` + +> See more: [`ops.Unit.set_workload_version`](https://ops.readthedocs.io/en/latest/#ops.Unit.set_workload_version) + +> Examples: [`jenkins-k8s` sets the workload version after getting it from the Jenkins package](https://github.com/canonical/jenkins-k8s-operator/blob/29e9b652714bd8314198965c41a60f5755dd381c/src/charm.py#L115), [`discourse-k8s` sets the workload version after getting it via an exec call](https://github.com/canonical/discourse-k8s-operator/blob/f523b29f909c69da7b9510b581dfcc2309698222/src/charm.py#L581), [`synapse` sets the workload version after getting it via an API call](https://github.com/canonical/synapse-operator/blob/778bcd414644c922373d542a304be14866835516/src/charm.py#L265) + +## Test the feature + +> See first: [Get started with charm testing](https://juju.is/docs/sdk/get-started-with-charm-testing) + +You'll want to add three levels of tests, unit, scenario, and integration. + + +### Write unit tests + +> See first: {ref}`write-unit-tests-for-a-charm` + +To verify the workload version is set in a unit test, use the +`ops.testing.Harness.get_workload_version()` method to +get the version that the charm set. In your `tests/unit/test_charm.py` file, +add a new test to verify the workload version is set; for example: + +```python +# You may already have this fixture to use in other tests. +@pytest.fixture() +def harness(): + yield ops.testing.Harness(MyCharm) + harness.cleanup() + +def test_start(harness): + # Suppose that the charm gets the workload version by running the command + # `/bin/server --version` in the container. Firstly, we mock that out: + harness.handle_exec("webserver", ["/bin/server", "--version"], result="1.2\n") + # begin_with_initial_hooks will trigger the 'start' event, and we expect + # the charm's 'start' handler to set the workload version. + harness.begin_with_initial_hooks() + assert harness.get_workload_version() == "1.2" +``` + +> See more: [`ops.testing.Harness.get_workload_version`](https://ops.readthedocs.io/en/latest/harness.html#ops.testing.Harness.get_workload_version) + +> Examples: [grafana-k8s checking the workload version](https://github.com/canonical/grafana-k8s-operator/blob/1c80f746f8edeae6fd23ddf31eed45f5b88c06b4/tests/unit/test_charm.py#L283) (and the [earlier mocking](https://github.com/canonical/grafana-k8s-operator/blob/1c80f746f8edeae6fd23ddf31eed45f5b88c06b4/tests/unit/test_charm.py#L127)), [sdcore-webui checks both that the version is set when it is available, and not set when not](https://github.com/canonical/sdcore-webui-k8s-operator/blob/1a66ad3f623d665657d04ad556139439f4733a28/tests/unit/test_charm.py#L447) + + +### Write scenario tests + +> See first: {ref}`write-scenario-tests-for-a-charm` + +To verify the workload version is set using Scenario, retrieve the workload +version from the `State`. In your `tests/scenario/test_charm.py` file, add a +new test that verifies the workload version is set. For example: + +```python +def test_workload_version_is_set(): + ctx = scenario.Context(MyCharm, meta={"name": "foo"}) + # Suppose that the charm gets the workload version by running the command + # `/bin/server --version` in the container. Firstly, we mock that out: + container = scenario.Container( + "webserver", + exec_mock={("/bin/server", "--version"): scenario.ExecOutput(stdout="1.2\n")}, + ) + out = ctx.run('start', scenario.State(containers={ref}`container])) + assert out.workload_version == "1.2" +``` + +### Write integration tests + +> See first: {ref}`write-integration-tests-for-a-charm` + +To verify that setting the workload version works correctly in an integration test, get the status +of the model, and check the `workload_version` attribute of the unit. In your +`tests/integration/test_charm.py` file, after the `test_build_and_deploy` test +that `charmcraft init` provides, add a new test that verifies the workload +version is set. For example: + +```python +# `charmcraft init` will provide you with this test. +async def test_build_and_deploy(ops_test: OpsTest): + # Build and deploy charm from local source folder + charm = await ops_test.build_charm(".") + + # Deploy the charm and wait for active/idle status + await asyncio.gather( + ops_test.model.deploy(charm, application_name=APP_NAME), + ops_test.model.wait_for_idle( + apps=[APP_NAME], status="active", raise_on_blocked=True, timeout=1000 + ), + ) + +async def test_workload_version_is_set(ops_test: OpsTest): + # Verify that the workload version has been set. + status = await ops_test.model.get_status() + version = status.applications[APP_NAME].units[f"{APP_NAME}/0"].workload_version + # We'll need to update this version every time we upgrade to a new workload + # version. If the workload has an API or some other way of getting the + # version, the test should get it from there and use that to compare to the + # unit setting. + assert version == "3.14" +``` + + + +> Examples: [synapse checking that the unit's workload version matches the one reported by the server](https://github.com/canonical/synapse-operator/blob/778bcd414644c922373d542a304be14866835516/tests/integration/test_charm.py#L139) + diff --git a/docs/howto/run-workloads-with-a-charm-kubernetes.md b/docs/howto/run-workloads-with-a-charm-kubernetes.md new file mode 100644 index 000000000..90d5d33e5 --- /dev/null +++ b/docs/howto/run-workloads-with-a-charm-kubernetes.md @@ -0,0 +1,938 @@ +(run-workloads-with-a-charm-kubernetes)= +# How to run workloads with a charm - Kubernetes + +The recommended way to create charms for Kubernetes is using the sidecar pattern with the workload container running Pebble. + +Pebble is a lightweight, API-driven process supervisor designed for use with charms. If you specify the `containers` field in a charm's `charmcraft.yaml`, Juju will deploy the charm code in a sidecar container, with Pebble running as the workload container's `ENTRYPOINT`. + +When the workload container starts up, Juju fires a [`PebbleReadyEvent`](https://ops.readthedocs.io/en/latest/#ops.PebbleReadyEvent), which can be handled using [`Framework.observe`](https://ops.readthedocs.io/en/latest/#ops.Framework.observe) as shown in {ref}`Framework Constructs under "Containers" <4554md>`. This gives the charm author access to `event.workload`, a [`Container`](https://ops.readthedocs.io/en/latest/#ops.Container) instance. + +The `Container` class has methods to modify the Pebble configuration "plan", start and stop services, read and write files, and run commands. These methods use the Pebble API, which communicates from the charm container to the workload container using HTTP over a Unix domain socket. + +The rest of this document provides details of how a charm interacts with the workload container via Pebble, using the Python Operator Framework [`Container`](https://ops.readthedocs.io/en/latest/#ops.Container) methods. + + +```{note} + +The [`Container.pebble`](https://ops.readthedocs.io/en/latest/#ops.Container.pebble) property returns the [`pebble.Client`](https://ops.readthedocs.io/en/latest/#ops.pebble.Client) instance for the given container. + +``` + +## Set up the workload container + +### Configure Juju to set up Pebble in the workload container + + + +The preferred way to run workloads on Kubernetes with charms is to start your workload with [Pebble](https://github.com/canonical/pebble). You do not need to modify upstream container images to make use of Pebble for managing your workload. The Juju controller automatically injects Pebble into workload containers using an [Init Container](https://kubernetes.io/docs/concepts/workloads/pods/init-containers/) and [Volume Mount](https://kubernetes.io/docs/concepts/storage/volumes/). The entrypoint of the container is overridden so that Pebble starts first and is able to manage running services. Charms communicate with the Pebble API using a UNIX socket, which is mounted into both the charm and workload containers. + +```{note} + +By default, you'll find the Pebble socket at `/var/lib/pebble/default/pebble.sock` in the workload container, and `/charm//pebble.sock` in the charm container. + +``` + +Most Kubernetes charms will need to define a `containers` map in their `charmcraft.yaml` in order to start a workload with a known OCI image: + +```yaml +# ... +containers: + myapp: + resource: myapp-image + redis: + resource: redis-image + +resources: + myapp-image: + type: oci-image + description: OCI image for my application + redis-image: + type: oci-image + description: OCI image for Redis +# ... +``` + +```{note} + +In some cases, you may wish not to specify a `containers` map, which will result in an "operator-only" charm. These can be useful when writing "integrator charms" (sometimes known as "proxy charms"), which are used to represent some external service in the Juju model. + +``` + +For each container, a resource of type `oci-image` must also be specified. The resource is used to inform the Juju controller how to find the correct OCI-compliant container image for your workload on Charmhub. + +If multiple containers are specified in `charmcraft.yaml` (as above), each Pod will contain an instance of every specified container. Using the example above, each Pod would be created with a total of 3 running containers: + +- a container running the `myapp-image` +- a container running the `redis-image` +- a container running the charm code + +The Juju controller emits [`PebbleReadyEvent`](https://ops.readthedocs.io/en/latest/#ops.PebbleReadyEvent)s to charms when Pebble has initialised its API in a container. These events are named `_pebble_ready`. Using the example above, the charm would receive two Pebble related events (assuming the Pebble API starts correctly in each workload): + +- `myapp_pebble_ready` +- `redis_pebble_ready`. + + +Consider the following example snippet from a `charmcraft.yaml`: + +```yaml +# ... +containers: + pause: + resource: pause-image + +resources: + pause-image: + type: oci-image + description: Docker image for google/pause +# ... +``` + +Once the containers are initialised, the charm needs to tell Pebble how to start the workload. Pebble uses a series of "layers" for its configuration. Layers contain a description of the processes to run, along with the path and arguments to the executable, any environment variables to be specified for the running process and any relevant process ordering (more information available in the Pebble [README](https://github.com/canonical/pebble)). + +```{note} + +In many cases, using the container's specified entrypoint may be desired. You can find the original entrypoint of an image locally like so: + +`$ docker pull ` +`$ docker inspect ` + +``` + +When using an OCI-image that is not built specifically for use with Pebble, layers are defined at runtime using Pebble’s API. Recall that when Pebble has initialised in a container (and the API is ready), the Juju controller emits a [`PebbleReadyEvent`](https://ops.readthedocs.io/en/latest/#ops.PebbleReadyEvent) event to the charm. Often it is in the callback bound to this event that layers are defined, and services started: + +```python +# ... +import ops +# ... + +class PauseCharm(ops.CharmBase): + # ... + def __init__(self, framework): + super().__init__(framework) + # Set a friendly name for your charm. This can be used with the Operator + # framework to reference the container, add layers, or interact with + # providers/consumers easily. + self.name = "pause" + # This event is dynamically determined from the service name + # in ops.pebble.Layer + # + # If you set self.name as above and use it in the layer definition following this + # example, the event will be _pebble_ready + framework.observe(self.on.pause_pebble_ready, self._on_pause_pebble_ready) + # ... + + def _on_pause_pebble_ready(self, event: ops.PebbleReadyEvent) -> None: + """Handle the pebble_ready event""" + # You can get a reference to the container from the PebbleReadyEvent + # directly with: + # container = event.workload + # + # The preferred method is through get_container() + container = self.unit.get_container(self.name) + # Add our initial config layer, combining with any existing layer + container.add_layer(self.name, self._pause_layer(), combine=True) + # Start the services that specify 'startup: enabled' + container.autostart() + self.unit.status = ops.ActiveStatus() + + def _pause_layer(self) -> ops.pebble.Layer: + """Returns Pebble configuration layer for google/pause""" + return ops.pebble.Layer( + { + "summary": "pause layer", + "description": "pebble config layer for google/pause", + "services": { + self.name: { + "override": "replace", + "summary": "pause service", + "command": "/pause", + "startup": "enabled", + } + }, + } + ) +# ... +``` + +A common method for configuring container workloads is by manipulating environment variables. The layering in Pebble makes this easy. Consider the following extract from a `config-changed` callback which combines a new overlay layer (containing some environment configuration) with the current Pebble layer and restarts the workload: + +```python +# ... +import ops +# ... +def _on_config_changed(self, event: ops.ConfigChangedEvent) -> None: + """Handle the config changed event.""" + # Get a reference to the container so we can manipulate it + container = self.unit.get_container(self.name) + + # Create a new config layer - specify 'override: merge' in + # the 'pause' service definition to overlay with existing layer + layer = ops.pebble.Layer( + { + "services": { + "pause": { + "override": "merge", + "environment": { + "TIMEOUT": self.model.config["timeout"], + }, + } + }, + } + ) + + try: + # Add the layer to Pebble + container.add_layer(self.name, layer, combine=True) + logging.debug("Added config layer to Pebble plan") + + # Tell Pebble to update the plan, which will restart any services if needed. + container.replan() + logging.info("Updated pause service") + # All is well, set an ActiveStatus + self.unit.status = ops.ActiveStatus() + except ops.pebble.PathError, ops.pebble.ProtocolError, ops.pebble.ConnectionError: + # handle errors (for example: the container might not be ready yet) + ..... +``` + +In this example, each time a `config-changed` event is fired, a new overlay layer is created that only includes the environment config, populated using the charm’s config. Pebble will ensure that that the application is only restarted if the configuration has changed. + +### Configure a Pebble layer + +Pebble services are [configured by means of layers](https://github.com/canonical/pebble#layer-specification), with higher layers adding to or overriding lower layers, forming the effective Pebble configuration, or "plan". + +When a workload container is created and Pebble starts up, it looks in `/var/lib/pebble/default/layers` (if that exists) for configuration layers already present in the container image, such as `001-layer.yaml`. If there are existing layers there, that becomes the starting configuration, otherwise Pebble is happy to start with an empty configuration, meaning no services. + +In the latter case, Pebble is configured dynamically via the API by adding layers at runtime. + +See the [layer specification](https://github.com/canonical/pebble#layer-specification) for more details. + + +#### Add a configuration layer + +To add a configuration layer, call [`Container.add_layer`](https://ops.readthedocs.io/en/latest/#ops.Container.add_layer) with a label for the layer, and the layer's contents as a YAML string, Python dict, or [`pebble.Layer`](https://ops.readthedocs.io/en/latest/#ops.pebble.Layer) object. + +You can see an example of `add_layer` under the ["Replan" heading](#heading--replan). The `combine=True` argument tells Pebble to combine the named layer into an existing layer of that name (or add a layer if none by that name exists). Using `combine=True` is common when dynamically adding layers. + +Because `combine=True` combines the layer with an existing layer of the same name, it's normally used with `override: replace` in the YAML service configuration. This means replacing the entire service configuration with the fields in the new layer. + +If you're adding a single layer with `combine=False` (default option) on top of an existing base layer, you may want to use `override: merge` in the service configuration. This will merge the fields specified with the service by that name in the base layer. [See an example of overriding a layer.](https://github.com/canonical/pebble#layer-override-example) + +#### Fetch the effective plan + +Charm authors can also introspect the current plan using [`Container.get_plan`](https://ops.readthedocs.io/en/latest/#ops.Container.get_plan). It returns a [`pebble.Plan`](https://ops.readthedocs.io/en/latest/#ops.pebble.Plan) object whose `services` attribute maps service names to [`pebble.Service`](https://ops.readthedocs.io/en/latest/#ops.pebble.Service) instances. + +Below is an example of how you might use `get_plan` to introspect the current configuration, and only add the layer with its services if they haven't been added already: + +```python +class MyCharm(ops.CharmBase): + ... + + def _on_config_changed(self, event): + container = self.unit.get_container("main") + plan = container.get_plan() + if not plan.services: + layer = {"services": ...} + container.add_layer("layer", layer) + container.start("svc") + ... +``` + + +## Control and monitor services in the workload container + +The main purpose of Pebble is to control and monitor services, which are usually long-running processes like web servers and databases. + +In the context of Juju sidecar charms, Pebble is run with the `--hold` argument, which prevents it from automatically starting the services marked with `startup: enabled`. This is to give the charm full control over when the services in Pebble's configuration are actually started. + +### Replan + +After adding a configuration layer to the plan (details below), you need to call `replan` to make any changes to `services` take effect. When you execute replan, Pebble will automatically restart any services that have changed, respecting dependency order. If the services are already running, it will stop them first using the normal [stop sequence](#heading--start-and-stop). + +The reason for replan is so that you as a user have control over when the (potentially high-impact) action of stopping and restarting your services takes place. + +Replan also starts the services that are marked as `startup: enabled` in the configuration plan, if they're not running already. + +Call [`Container.replan`](https://ops.readthedocs.io/en/latest/#ops.Container.replan) to execute the replan procedure. For example: + +```python +class SnappassTestCharm(ops.CharmBase): + ... + + def _start_snappass(self): + container = self.unit.containers["snappass"] + snappass_layer = { + "services": { + "snappass": { + "override": "replace", + "summary": "snappass service", + "command": "snappass", + "startup": "enabled", + } + }, + } + container.add_layer("snappass", snappass_layer, combine=True) + container.replan() + self.unit.status = ops.ActiveStatus() +``` + +### Check container health + +The Ops library provides a way to ensure that your container is healthy. In the `Container` class, `Container.can_connect()` can be used if you only need to know that Pebble is responding at a specific point in time - for example to update a status message. This should *not* be used to guard against later Pebble operations, because that introduces a race condition where Pebble might be responsive when `can_connect()` is called, but is not when the later operation is executed. Instead, charms should always include `try`/`except` statements around Pebble operations, to avoid the unit going into error state. + +> See more: [`ops.Container`](https://ops.readthedocs.io/en/latest/#ops.Container) + +### Start and stop + +To start (or stop) one or more services by name, use the [`start`](https://ops.readthedocs.io/en/latest/#ops.Container.start) and [`stop`](https://ops.readthedocs.io/en/latest/#ops.Container.stop) methods. Here's an example of how you might stop and start a database service during a backup action: + +```python +class MyCharm(ops.CharmBase): + ... + + def _on_pebble_ready(self, event): + container = event.workload + container.start('mysql') + + def _on_backup_action(self, event): + container = self.unit.get_container('main') + try: + container.stop('mysql') + do_mysql_backup() + container.start('mysql') + except ops.pebble.ProtocolError, ops.pebble.PathError, ops.pebble.ConnectionError: + # handle Pebble errors +``` + +It's not an error to start a service that's already started, or stop one that's already stopped. These actions are *idempotent*, meaning they can safely be performed more than once, and the service will remain in the same state. + +When Pebble starts a service, Pebble waits one second to ensure the process doesn't exit too quickly -- if the process exits within one second, the start operation raises an error and the service remains stopped. + +To stop a service, Pebble first sends `SIGTERM` to the service's process group to try to stop the service gracefully. If the process has not exited after 5 seconds, Pebble sends `SIGKILL` to the process group. If the process still doesn't exit after another 5 seconds, the stop operation raises an error. If the process exits any time before the 10 seconds have elapsed, the stop operation succeeds. + +### Fetch service status + +You can use the [`get_service`](https://ops.readthedocs.io/en/latest/#ops.Container.get_service) and [`get_services`](https://ops.readthedocs.io/en/latest/#ops.Container.get_services) methods to fetch the current status of one service or multiple services, respectively. The returned [`ServiceInfo`](https://ops.readthedocs.io/en/latest/#ops.pebble.ServiceInfo) objects provide a `status` attribute with various states, or you can use the [`ServiceInfo.is_running`](https://ops.readthedocs.io/en/latest/#ops.pebble.ServiceInfo.is_running) method. + +Here is a modification to the start/stop example that checks whether the service is running before stopping it: + +```python +class MyCharm(ops.CharmBase): + ... + + def _on_backup_action(self, event): + container = self.unit.get_container('main') + is_running = container.get_service('mysql').is_running() + if is_running: + container.stop('mysql') + do_mysql_backup() + if is_running: + container.start('mysql') +``` + +### Send signals to services + +From Juju version 2.9.22, you can use the [`Container.send_signal`](https://ops.readthedocs.io/en/latest/#ops.Container.send_signal) method to send a signal to one or more services. For example, to send `SIGHUP` to the hypothetical "nginx" and "redis" services: + +```python +container.send_signal('SIGHUP', 'nginx', 'redis') +``` + +This will raise an `APIError` if any of the services are not in the plan or are not currently running. + +### View service logs + +Pebble stores service logs (stdout and stderr from services) in a ring buffer accessible via the `pebble logs` command. Each log line is prefixed with the timestamp and service name, using the format `2021-05-03T03:55:49.654Z [snappass] ...`. Pebble allocates a ring buffer of 100KB per service (not one ring to rule them all), and overwrites the oldest logs in the buffer when it fills up. + +When running under Juju, the Pebble server is started with the `--verbose` flag, which means it also writes these logs to Pebble's own stdout. That in turn is accessible via Kubernetes using the `kubectl logs` command. For example, to view the logs for the "redis" container, you could run: + +``` +microk8s kubectl logs -n snappass snappass-test-0 -c redis +``` + +In the command line above, "snappass" is the namespace (Juju model name), "snappass-test-0" is the pod, and "redis" the specific container defined by the charm configuration. + +### Configure service auto-restart + +From Juju version 2.9.22, Pebble automatically restarts services when they exit unexpectedly. + +By default, Pebble will automatically restart a service when it exits (with either a zero or nonzero exit code). In addition, Pebble implements an exponential backoff delay and a small random jitter time between restarts. + +You can configure this behavior in the layer configuration, specified under each service. Here is an example showing the complete list of auto-restart options with their defaults: + +```yaml +services: + server: + override: replace + command: python3 app.py + + # auto-restart options (showing defaults) + on-success: restart # can also be "shutdown" or "ignore" + on-failure: restart # can also be "shutdown" or "ignore" + backoff-delay: 500ms + backoff-factor: 2.0 + backoff-limit: 30s +``` + +The `on-success` action is performed if the service exits with a zero exit code, and the `on-failure` action is performed if it exits with a nonzero code. The actions are defined as follows: + +* `restart`: automatically restart the service after the current backoff delay. This is the default. +* `shutdown`: shut down the Pebble server. Because Pebble is the container's "PID 1" process, this will cause the container to terminate -- useful if you want Kubernetes to restart the container. +* `ignore`: do nothing (apart from logging the failure). + +The backoff delay between restarts is calculated using an exponential backoff: `next = current * backoff_factor`, with `current` starting at the configured `backoff-delay`. If `next` is greater than `backoff-limit`, it is capped at `backoff-limit`. With the defaults, the delays (in seconds) will be: 0.5, 1, 2, 4, 8, 16, 30, 30, and so on. + +The `backoff-factor` must be greater than or equal to 1.0. If the factor is set to 1.0, `next` will equal `current`, so the delay will remain constant. + +Just before delaying, a small random time jitter of 0-10% of the delay is added (the current delay is not updated). For example, if the current delay value is 2 seconds, the actual delay will be between 2.0 and 2.2 seconds. + +## Perform health checks on the workload container + +From Juju version 2.9.26, Pebble supports adding custom health checks: first, to allow Pebble itself to restart services when certain checks fail, and second, to allow Kubernetes to restart containers when specified checks fail. + +Each check can be one of three types. The types and their success criteria are: + +* `http`: an HTTP `GET` request to the URL specified must return an HTTP 2xx status code. +* `tcp`: opening the given TCP port must be successful. +* `exec`: executing the specified command must yield a zero exit code. + + +### Check configuration + +Checks are configured in the layer configuration using the top-level field `checks`. Here's an example showing the three different types of checks: + +```yaml +checks: + up: + override: replace + level: alive # optional, but required for liveness/readiness probes + period: 10s # this is the default + timeout: 3s # this is the default + threshold: 3 # this is the default + exec: + command: service nginx status + + online: + override: replace + level: ready + tcp: + port: 8080 + + http-test: + override: replace + http: + url: http://localhost:8080/test +``` + +Each check is performed with the specified `period` (the default is 10 seconds apart), and is considered an error if a `timeout` happens before the check responds -- for example, before the HTTP request is complete or before the command finishes executing. + +A check is considered healthy until it's had `threshold` errors in a row (the default is 3). At that point, the `on-check-failure` action will be triggered, and the health endpoint will return an error response (both are discussed below). When the check succeeds again, the failure count is reset. + +See the [layer specification](https://github.com/canonical/pebble#layer-specification) for more details about the fields and options for different types of checks. + +### Respond to a check failing or recovering + +> Added in `ops 2.15` and `juju 3.6` + + +To have the charm respond to a check reaching the failure threshold, or passing again afterwards, observe the `pebble_check_failed` and `pebble_check_recovered` events and switch on the info's `name`: + +```python +class PostgresCharm(ops.CharmBase): + def __init__(self, framework: ops.Framework): + super().__init__(framework) + # Note that "db" is the workload container's name + framework.observe(self.on["db"].pebble_check_failed, self._on_pebble_check_failed) + framework.observe(self.on["db"].pebble_check_recovered, self._on_pebble_check_recovered) + + def _on_pebble_check_failed(self, event: ops.PebbleCheckFailedEvent): + if event.info.name == "http-test": + logger.warning("The http-test has started failing!") + self.unit.status = ops.ActiveStatus("Degraded functionality ...") + + elif event.info == "online": + logger.error("The service is no longer online!") + + def _on_pebble_check_recovered(self, event: ops.PebbleCheckRecoveredEvent): + if event.info.name == "http-test": + logger.warning("The http-test has stopped failing!") + self.unit.status = ops.ActiveStatus() + + elif event.info == "online": + logger.error("The service is online again!") +``` + +All check events have an `info` property with the details of the check's current status. Note that, by the time that the charm receives the event, the status of the check may have changed (for example, passed again after failing). If the response to the check failing is light (such as changing the status), then it's fine to rely on the status of the check at the time the event was triggered — there will be a subsequent check-recovered event, and the status will quickly flick back to the correct one. If the response is heavier (such as restarting a service with an adjusted configuration), then the two events should share a common handler and check the current status via the `info` property; for example: + +```python +class PostgresCharm(ops.CharmBase): + def __init__(self, framework: ops.Framework): + super().__init__(framework) + # Note that "db" is the workload container's name + framework.observe(self.on["db"].pebble_check_failed, self._on_pebble_check_failed) + framework.observe(self.on["db"].pebble_check_recovered, self._on_pebble_check_recovered) + + def _on_pebble_check_failed(self, event: ops.PebbleCheckFailedEvent): + if event.info.name != "up": + # For now, we ignore the other tests. + return + if event.info.status == ops.pebble.CheckStatus.DOWN: + self.activate_alternative_configuration() + else: + self.activate_main_configuration() +``` + +### Fetch check status + +You can use the [`get_check`](https://ops.readthedocs.io/en/latest/#ops.Container.get_check) and [`get_checks`](https://ops.readthedocs.io/en/latest/#ops.Container.get_checks) methods to fetch the current status of one check or multiple checks, respectively. The returned [`CheckInfo`](https://ops.readthedocs.io/en/latest/#ops.pebble.CheckInfo) objects provide various attributes, most importantly a `status` attribute which will be either `UP` or `DOWN`. + +Here is a code example that checks whether the `uptime` check is healthy, and writes an error log if not: + +```python +container = self.unit.get_container('main') +check = container.get_check('uptime') +if check.status != ops.pebble.CheckStatus.UP: + logger.error('Uh oh, uptime check unhealthy: %s', check) +``` + +### Check auto-restart + +To enable Pebble auto-restart behavior based on a check, use the `on-check-failure` map in the service configuration. For example, to restart the "server" service when the "http-test" check fails, use the following configuration: + +```yaml +services: + server: + override: merge + on-check-failure: + http-test: restart # can also be "shutdown" or "ignore" (the default) +``` + +### Check health endpoint and probes + +As of Juju version 2.9.26, Pebble includes an HTTP `/v1/health` endpoint that allows a user to query the health of configured checks, optionally filtered by check level with the query string `?level=` This endpoint returns an HTTP 200 status if the checks are healthy, HTTP 502 otherwise. + +Each check can optionally specify a `level` of "alive" or "ready". These have semantic meaning: "alive" means the check or the service it's connected to is up and running; "ready" means it's properly accepting network traffic. These correspond to Kubernetes ["liveness" and "readiness" probes](https://kubernetes.io/docs/tasks/configure-pod-container/configure-liveness-readiness-startup-probes/). + +When Juju creates a sidecar charm container, it initialises the Kubernetes liveness and readiness probes to hit the `/v1/health` endpoint with `?level=alive` and `?level=ready` filters, respectively. + +Ready implies alive, and not alive implies not ready. If you've configured an "alive" check but no "ready" check, and the "alive" check is unhealthy, `/v1/health?level=ready` will report unhealthy as well, and the Kubernetes readiness probe will act on that. + +If there are no checks configured, Pebble returns HTTP 200 so the liveness and readiness probes are successful by default. To use this feature, you must explicitly create checks with `level: alive` or `level: ready` in the layer configuration. + +Consider the K8s liveness success (`level=alive` check) to mean "Pebble is alive" rather than "the application is fully alive" (and failure to mean "this container needs to die"). For charms that take a long time to start, you should not have a `level=alive` check (if Pebble's running, it will report alive to K8s), and instead use an ordinary Pebble check (without a `level`) in conjunction with `on-check-failure: restart`. That way Pebble itself has full control over restarting the service in question. + +### Test checks + +> Added in Scenario 7.0 + +To test charms that use Pebble check events, use the Scenario `CheckInfo` class and the emit the appropriate event. For example, to simulate the "http-test" check failing, the charm test could do the following: + +```python +def test_http_check_failing(): + ctx = scenario.Context(PostgresCharm) + check_info = scenario.CheckInfo("http-test", failures=3, status=ops.pebble.CheckStatus.DOWN) + container = scenario.Container("db", check_infos={check_info}) + state_in = scenario.State(containers={container}) + + state_out = ctx.run(ctx.on.pebble_check_failed(container, check_info), state_in) + + assert state_out... +``` + +## Manage files in the workload container + +Pebble's files API allows charm authors to read and write files on the workload container. You can write files ("push"), read files ("pull"), list files in a directory, make directories, and delete files or directories. + + +### Push + +Probably the most useful operation is [`Container.push`](https://ops.readthedocs.io/en/latest/#ops.Container.push), which allows you to write a file to the workload, for example, a PostgreSQL configuration file. You can use `push` as follows (note that this code would be inside a charm event handler): + +```python +config = """ +port = 7777 +max_connections = 1000 +""" +container.push('/etc/pg/postgresql.conf', config, make_dirs=True) +``` + +The `make_dirs=True` flag tells `push` to create the intermediate directories if they don't already exist (`/etc/pg` in this case). + +There are many additional features, including the ability to send raw bytes (by providing a Python `bytes` object as the second argument) and write data from a file-like object. You can also specify permissions and the user and group for the file. See the [API documentation](https://ops.readthedocs.io/en/latest/#ops.Container.push) for details. + +### Pull + +To read a file from the workload, use [`Container.pull`](https://ops.readthedocs.io/en/latest/#ops.Container.pull), which returns a file-like object that you can `read()`. + +The files API doesn't currently support update, so to update a file you can use `pull` to perform a read-modify-write operation, for example: + +```python +# Update port to 8888 and restart service +config = container.pull('/etc/pg/postgresql.conf').read() +if 'port =' not in config: + config += '\nport = 8888\n' +container.push('/etc/pg/postgresql.conf', config) +container.restart('postgresql') +``` + +If you specify the keyword argument `encoding=None` on the `pull()` call, reads from the returned file-like object will return `bytes`. The default is `encoding='utf-8'`, which will decode the file's bytes from UTF-8 so that reads return a Python `str`. + +### Push recursive + +> Added in 1.5 + +To copy several files to the workload, use [`Container.push_path`](https://ops.readthedocs.io/en/latest/#ops.Container.push_path), which copies files recursively into a specified destination directory. The API docs contain detailed examples of source and destination semantics and path handling. + +```python +# copy "/source/dir/[files]" into "/destination/dir/[files]" +container.push_path('/source/dir', '/destination') + +# copy "/source/dir/[files]" into "/destination/[files]" +container.push_path('/source/dir/*', '/destination') +``` + +A trailing "/*" on the source directory is the only supported globbing/matching. + +### Pull recursive + +> Added in 1.5 + +To copy several files to the workload, use [`Container.pull_path`](https://ops.readthedocs.io/en/latest/#ops.Container.pull_path), which copies files recursively into a specified destination directory. The API docs contain detailed examples of source and destination semantics and path handling. + +```python +# copy "/source/dir/[files]" into "/destination/dir/[files]" +container.pull_path('/source/dir', '/destination') + +# copy "/source/dir/[files]" into "/destination/[files]" +container.pull_path('/source/dir/*', '/destination') +``` + +A trailing "/*" on the source directory is the only supported globbing/matching. + +### List files + +To list the contents of a directory or return stat-like information about one or more files, use [`Container.list_files`](https://ops.readthedocs.io/en/latest/#ops.Container.list_files). It returns a list of [`pebble.FileInfo`](https://ops.readthedocs.io/en/latest/#ops.pebble.FileInfo) objects for each entry (file or directory) in the given path, optionally filtered by a glob pattern. For example: + +```python +infos = container.list_files('/etc', pattern='*.conf') +total_size = sum(f.size for f in infos) +logger.info('total size of config files: %d', total_size) +names = set(f.name for f in infos) +if 'host.conf' not in names: + raise Exception('This charm requires /etc/host.conf!') +``` + +If you want information about the directory itself (instead of its contents), call `list_files(path, itself=True)`. + +### Create directory + +To create a directory, use [`Container.make_dir`](https://ops.readthedocs.io/en/latest/#ops.Container.make_dir). It takes an optional `make_parents=True` argument (like `mkdir -p`), as well as optional permissions and user/group arguments. Some examples: + +```python +container.make_dir('/etc/pg', user='postgres', group='postgres') +container.make_dir('/some/other/nested/dir', make_parents=True) +``` + +### Remove path + +To delete a file or directory, use [`Container.remove_path`](https://ops.readthedocs.io/en/latest/#ops.Container.remove_path). If a directory is specified, it must be empty unless `recursive=True` is specified, in which case the entire directory tree is deleted, recursively (like `rm -r`). For example: + +```python +# Delete Apache access log +container.remove_path('/var/log/apache/access.log') +# Blow away /tmp/mysubdir and all files under it +container.remove_path('/tmp/mysubdir', recursive=True) +``` + +### Check file and directory existence + +> Added in 1.4 + +To check if a paths exists you can use [`Container.exists`](https://ops.readthedocs.io/en/latest/#ops.Container.exists) for directories or files and [`Container.isdir`](https://ops.readthedocs.io/en/latest/#ops.Container.isdir) for directories. These functions are analogous to python's `os.path.isdir` and `os.path.exists` functions. For example: + +```python +# if /tmp/myfile exists +container.exists('/tmp/myfile') # True +container.isdir('/tmp/myfile') # False + +# if /tmp/mydir exists +container.exists('/tmp/mydir') # True +container.isdir('/tmp/mydir') # True +``` + +## Run commands on the workload container + +From Juju 2.9.17, Pebble includes an API for executing arbitrary commands on the workload container: the [`Container.exec`](https://ops.readthedocs.io/en/latest/#ops.Container.exec) method. It supports sending stdin to the process and receiving stdout and stderr, as well as more advanced options. + +To run simple commands and receive their output, call `Container.exec` to start the command, and then use the returned [`Process`](https://ops.readthedocs.io/en/latest/#ops.pebble.ExecProcess) object's [`wait_output`](https://ops.readthedocs.io/en/latest/#ops.pebble.ExecProcess.wait_output) method to wait for it to finish and collect its output. + +For example, to back up a PostgreSQL database, you might use `pg_dump`: + +```python +process = container.exec(['pg_dump', 'mydb'], timeout=5*60) +sql, warnings = process.wait_output() +if warnings: + for line in warnings.splitlines(): + logger.warning('pg_dump: %s', line.strip()) +# do something with "sql" +``` + + +### Handle errors + +The `exec` method raises a [`pebble.APIError`](https://ops.readthedocs.io/en/latest/#ops.pebble.APIError) if basic checks fail and the command can't be executed at all, for example, if the executable is not found. + +The [`ExecProcess.wait`](https://ops.readthedocs.io/en/latest/#ops.pebble.ExecProcess.wait) and [`ExecProcess.wait_output`](https://ops.readthedocs.io/en/latest/#ops.pebble.ExecProcess.wait_output) methods raise [`pebble.ChangeError`](https://ops.readthedocs.io/en/latest/#ops.pebble.ChangeError) if there was an error starting or running the process, and [`pebble.ExecError`](https://ops.readthedocs.io/en/latest/#ops.pebble.ExecError) if the process exits with a non-zero exit code. + +In the case where the process exits via a signal (such as SIGTERM or SIGKILL), the exit code will be 128 plus the signal number. SIGTERM's signal number is 15, so a process terminated via SIGTERM would give exit code 143 (128+15). + +It's okay to let these exceptions bubble up: Juju will mark the hook as failed and re-run it automatically. However, if you want fine-grained control over error handling, you can catch the `ExecError` and inspect its attributes. For example: + +```python +process = container.exec(['cat', '--bad-arg']) +try: + stdout, _ = process.wait_output() + logger.info(stdout) +except ops.pebble.ExecError as e: + logger.error('Exited with code %d. Stderr:', e.exit_code) + for line in e.stderr.splitlines(): + logger.error(' %s', line) +``` + +That will log something like this: + +```text +Exited with code 1. Stderr: + cat: unrecognized option '--bad-arg' + Try 'cat --help' for more information. +``` + +### Use command options + +The `Container.exec` method has various options (see [full API documentation](https://ops.readthedocs.io/en/latest/#ops.pebble.Client.exec)), including: + +* `environment`: a dict of environment variables to pass to the process +* `working_dir`: working directory to run the command in +* `timeout`: command timeout in seconds +* `user_id`, `user`, `group_id`, `group`: UID/username and GID/group name to run command as +* `service_context`: run the command in the context of the specified service + +Here is a (contrived) example showing the use of most of these parameters: + +```python +process = container.exec( + ['/bin/sh', '-c', 'echo HOME=$HOME, PWD=$PWD, FOO=$FOO'], + environment={'FOO': 'bar'}, + working_dir='/tmp', + timeout=5.0, + user='bob', + group='staff', +) +stdout, _ = process.wait_output() +logger.info('Output: %r', stdout) +``` + +This will execute the echo command in a shell and log something like `Output: 'HOME=/home/bob, PWD=/tmp, FOO=bar\n'`. + +The `service_context` option allows you to specify the name of a service to "inherit" context from. Specifically, inherit its environment variables, user/group settings, and working directory. The other exec options (`user_id`, `user`, `group_id`, `group`, `working_dir`) will override the service's settings; `environment` will be merged on top of the service’s environment. + +Here's an example that uses the `service_context` option: + +```python +# Use environment, user/group, and working_dir from "database" service +process = container.exec(['pg_dump', 'mydb'], service_context='database') +process.wait_output() +``` + +### Use input/output options + +The simplest way of receiving standard output and standard error is by using the [`ExecProcess.wait_output`](https://ops.readthedocs.io/en/latest/#ops.pebble.ExecProcess.wait_output) method as shown below. The simplest way of sending standard input to the program is as a string, using the `stdin` parameter to `exec`. For example: + +```python +process = container.exec(['tr', 'a-z', 'A-Z'], + stdin='This is\na test\n') +stdout, _ = process.wait_output() +logger.info('Output: %r', stdout) +``` + +By default, input is sent and output is received as Unicode using the UTF-8 encoding. You can change this with the `encoding` parameter (which defaults to `utf-8`). The most common case is to set `encoding=None`, which means "use raw bytes", in which case `stdin` must be a bytes object and `wait_output()` returns bytes objects. + +For example, the following will log `Output: b'\x01\x02'`: + +```python +process = container.exec(['cat'], stdin=b'\x01\x02', + encoding=None) +stdout, _ = process.wait_output() +logger.info('Output: %r', stdout) +``` + +You can also pass [file-like objects](https://docs.python.org/3/glossary.html#term-file-object) using the `stdin`, `stdout`, and `stderr` parameters. These can be real files, streams, `io.StringIO` instances, and so on. When the `stdout` and `stderr` parameters are specified, call the `ExecProcess.wait` method instead of `wait_output`, as output is being written, not returned. + +For example, to pipe standard input from a file to the command, and write the result to a file, you could use the following: + +```python +with open('LICENSE.txt') as stdin: + with open('output.txt', 'w') as stdout: + process = container.exec( + ['tr', 'a-z', 'A-Z'], + stdin=stdin, + stdout=stdout, + stderr=sys.stderr, + ) + process.wait() +# use result in "output.txt" +``` + +For advanced uses, you can also perform streaming I/O by reading from and writing to the `stdin` and `stdout` attributes of the `ExecProcess` instance. For example, to stream lines to a process and log the results as they come back, use something like the following: + +```python +process = container.exec(['cat']) + +# Thread that sends data to process's stdin +def stdin_thread(): + try: + for line in ['one\n', '2\n', 'THREE\n']: + process.stdin.write(line) + process.stdin.flush() + time.sleep(1) + finally: + process.stdin.close() +threading.Thread(target=stdin_thread).start() + +# Log from stdout stream as output is received +for line in process.stdout: + logging.info('Output: %s', line.strip()) + +# Will return immediately as stdin was closed above +process.wait() +``` + +That will produce the following logs: + +``` +Output: 'one\n' +Output: '2\n' +Output: 'THREE\n' +``` + +Caution: it's easy to get threading wrong and cause deadlocks, so it's best to use `wait_output` or pass file-like objects to `exec` instead if possible. + +### Send signals to a running command + +To send a signal to the running process, use [`ExecProcess.send_signal`](https://ops.readthedocs.io/en/latest/#ops.pebble.ExecProcess.send_signal) with a signal number or name. For example, the following will terminate the "sleep 10" process after one second: + +```python +process = container.exec({ref}`'sleep', '10']) +time.sleep(1) +process.send_signal(signal.SIGTERM) +process.wait() +``` + +Note that because sleep will exit via a signal, `wait()` will raise an `ExecError` with an exit code of 143 (128+SIGTERM): + +``` +Traceback (most recent call last): + .. +ops.pebble.ExecError: non-zero exit code 143 executing ['sleep', '10'] +``` + +## Use custom notices from the workload container + +### Record a notice + +To record a custom notice, use the `pebble notify` CLI command. For example, the workload might have a script to back up the database and then record a notice: + +```sh +pg_dump mydb >/tmp/mydb.sql +/charm/bin/pebble notify canonical.com/postgresql/backup-done path=/tmp/mydb.sql +``` + +The first argument to `pebble notify` is the key, which must be in the format `/`. The caller can optionally provide map data arguments in `=` format; this example shows a single data argument named `path`. + +The `pebble notify` command has an optional `--repeat-after` flag, which tells Pebble to only allow the notice to repeat after the specified duration (the default is to repeat for every occurrence). If the caller says `--repeat-after=1h`, Pebble will prevent the notice with the same type and key from repeating within an hour -- useful to avoid the charm waking up too often when a notice occcurs frequently. + +> See more: [GitHub | Pebble > Notices > `pebble notify`](https://github.com/canonical/pebble#notices) + +### Respond to a notice + +To have the charm respond to a notice, observe the `pebble_custom_notice` event and switch on the notice's `key`: + +```python +class PostgresCharm(ops.CharmBase): + def __init__(self, framework: ops.Framework): + super().__init__(framework) + # Note that "db" is the workload container's name + framework.observe(self.on["db"].pebble_custom_notice, self._on_pebble_custom_notice) + + def _on_pebble_custom_notice(self, event: ops.PebbleCustomNoticeEvent) -> None: + if event.notice.key == "canonical.com/postgresql/backup-done": + path = event.notice.last_data["path"] + logger.info("Backup finished, copying %s to the cloud", path) + f = event.workload.pull(path, encoding=None) + s3_bucket.upload_fileobj(f, "db-backup.sql") + + elif event.notice.key == "canonical.com/postgresql/other-thing": + logger.info("Handling other thing") +``` + +All notice events have a [`notice`](https://ops.readthedocs.io/en/latest/#ops.PebbleNoticeEvent.notice) property with the details of the notice recorded. That is used in the example above to switch on the notice `key` and look at its `last_data` (to determine the backup's path). + +### Fetch notices + +A charm can also query for notices using the following two `Container` methods: + +* [`get_notice`](https://ops.readthedocs.io/en/latest/#ops.Container.get_notice), which gets a single notice by unique ID (the value of `notice.id`). +* [`get_notices`](https://ops.readthedocs.io/en/latest/#ops.Container.get_notices), which returns all notices by default, and allows filtering notices by specific attributes such as `key`. + +### Test notices + +To test charms that use Pebble Notices, use the [`Harness.pebble_notify`](https://ops.readthedocs.io/en/latest/#ops.testing.Harness.pebble_notify) method to simulate recording a notice with the given details. For example, to simulate the "backup-done" notice handled above, the charm tests could do the following: + +```python +class TestCharm(unittest.TestCase): + @patch("charm.s3_bucket.upload_fileobj") + def test_backup_done(self, upload_fileobj): + harness = ops.testing.Harness(PostgresCharm) + self.addCleanup(harness.cleanup) + harness.begin() + harness.set_can_connect("db", True) + + # Pretend backup file has been written + root = harness.get_filesystem_root("db") + (root / "tmp").mkdir() + (root / "tmp" / "mydb.sql").write_text("BACKUP") + + # Notify to record the notice and fire the event + harness.pebble_notify( + "db", "canonical.com/postgresql/backup-done", data={"path": "/tmp/mydb.sql"} + ) + + # Ensure backup content was "uploaded" to S3 + upload_fileobj.assert_called_once() + upload_f, upload_key = upload_fileobj.call_args.args + self.assertEqual(upload_f.read(), b"BACKUP") + self.assertEqual(upload_key, "db-backup.sql") +``` + + diff --git a/docs/howto/run-workloads-with-a-charm-machines.md b/docs/howto/run-workloads-with-a-charm-machines.md new file mode 100644 index 000000000..0e68ee858 --- /dev/null +++ b/docs/howto/run-workloads-with-a-charm-machines.md @@ -0,0 +1,81 @@ +(run-workloads-with-a-charm-machines)= +# How to run workloads with a charm - machines + +There are several ways your charm might start a workload, depending on the type of charm you’re authoring. + + +For a machine charm, it is likely that packages will need to be fetched, installed and started to provide the desired charm functionality. This can be achieved by interacting with the system’s package manager, ensuring that package and service status is maintained by reacting to events accordingly. + +It is important to consider which events to respond to in the context of your charm. A simple example might be: + +```python +# ... +from subprocess import check_call, CalledProcessError +# ... +class MachineCharm(ops.CharmBase): + #... + + def __init__(self, *args): + super().__init__(*args) + self.framework.observe(self.on.install, self._on_install) + self.framework.observe(self.on.start, self._on_start) + # ... + + def _on_install(self, event: ops.InstallEvent) -> None: + """Handle the install event""" + try: + # Install the openssh-server package using apt-get + check_call(["apt-get", "install", "-y", "openssh-server"]) + except ops.CalledProcessError as e: + # If the command returns a non-zero return code, put the charm in blocked state + logger.debug("Package install failed with return code %d", e.returncode) + self.unit.status = ops.BlockedStatus("Failed to install packages") + + def _on_start(self, event: ops.StartEvent) -> None: + """Handle the start event""" + try: + # Enable the ssh systemd unit, and start it + check_call(["systemctl", "enable", "--now", "openssh-server"]) + except ops.CalledProcessError as e: + # If the command returns a non-zero return code, put the charm in blocked state + logger.debug("Starting systemd unit failed with return code %d", e.returncode) + self.unit.status = ops.BlockedStatus("Failed to start/enable ssh service") + return + + # Everything is awesome + self.unit.status = ops.ActiveStatus() +``` + +If the machine is likely to be long-running and endure multiple upgrades throughout its life, it may be prudent to ensure the package is installed more regularly, and handle the case where it needs upgrading or reinstalling. Consider this excerpt from the [ubuntu-advantage charm code](https://git.launchpad.net/charm-ubuntu-advantage/tree/src/charm.py) (with some additional comments): + +```python +class UbuntuAdvantageCharm(ops.CharmBase): + """Charm to handle ubuntu-advantage installation and configuration""" + _state = ops.StoredState() + + def __init__(self, *args): + super().__init__(*args) + self._state.set_default(hashed_token=None, package_needs_installing=True, ppa=None) + self.framework.observe(self.on.config_changed, self.config_changed) + + def config_changed(self, event): + """Install and configure ubuntu-advantage tools and attachment""" + logger.info("Beginning config_changed") + self.unit.status = ops.MaintenanceStatus("Configuring") + # Helper method to ensure a custom PPA from charm config is present on the system + self._handle_ppa_state() + # Helper method to ensure latest package is installed + self._handle_package_state() + # Handle some ubuntu-advantage specific configuration + self._handle_token_state() + # Set the unit status using a helper _handle_status_state + if isinstance(self.unit.status, ops.BlockedStatus): + return + self._handle_status_state() + logger.info("Finished config_changed") + +``` + +In the example above, the package install status is ensured each time the charm's `config-changed` event fires, which should ensure correct state throughout the charm's deployed lifecycle. diff --git a/docs/howto/turn-a-hooks-based-charm-into-an-ops-charm.md b/docs/howto/turn-a-hooks-based-charm-into-an-ops-charm.md new file mode 100644 index 000000000..82ea42969 --- /dev/null +++ b/docs/howto/turn-a-hooks-based-charm-into-an-ops-charm.md @@ -0,0 +1,302 @@ +(turn-a-hooks-based-charm-into-an-ops-charm)= +# How to turn a hooks-based charm into an ops charm + +> See first: [`juju` | Charm taxonomy]() + +Suppose you have a hooks-based charm and you decide to rewrite it using the Ops framework in Python. + +The core concept tying hooks to the Ops framework is that hooks are no longer scripts stored in files that are named like the event they are meant to respond to; hooks are, instead, blocks of Python code that are best written as methods of a class. The class represents the charm; the methods, the operator logic to be executed as specific events occur. + +Here we'll look at just how to do that. You will learn how to: + - look at a simple hooks-based charm to understand its relationship with the Juju state machine; +- map that forward to the Ops framework; +- translate some shell commands to their Ops counterparts; +- translate some more shell commands by using some handy charm libraries. + + +This guide will refer to a local LXD cloud and a machine charm, but you can easily generalize the approach to Kubernetes. + + +## Analyse the charm + + +We start by looking at the charm we intend to translate; as an example, we will take [microsample](https://github.com/erik78se/charm-microsample), an educational charm, because it is simple and includes a number of hooks, while implementing little-to-no business logic (the charm does very little). + +From the charm root directory we see: + +```bash +$ tree . +. +├── charmcraft.yaml +├── config.yaml +├── copyright +├── hooks +│ ├── config-changed +│ ├── install +│ ├── start +│ ├── stop +│ ├── update-status +│ ├── upgrade-charm +│ ├── website-relation-broken +│ ├── website-relation-changed +│ ├── website-relation-departed +│ └── website-relation-joined +├── icon.svg +├── LICENSE +├── metadata.yaml +├── microsample-ha.png +├── README.md +└── revision +``` + +By looking at the `hooks` folder, we can already tell that there are two categories of hooks we'll need to port; +- core lifecycle hooks: + - `config-changed` + - `install` + - `start` + - `stop` + - `update-status` + - `upgrade-charm` +- hooks related to a `website` relation: + - `website-relation-*` + +If we look at `metadata.yaml` in fact we'll see: +```yaml +provides: + website: + interface: http +``` + +### Setting up the stage + +If we look at `charmcraft.yaml`, we'll see a section: +```yaml +parts: + microsample: + plugin: dump + source: . + prime: + - LICENSE + - README.md + - config.yaml + - copyright + - hooks + - icon.svg + - metadata.yaml +``` +This is a spec required to make charmcraft work in 'legacy mode' and support older charm frameworks, such as the hooks charm we are working with. +As such, if we take a look at the packed `.charm` file, we'll see that the files and folders listed in 'prime' are copied over one-to-one in the archive. + +If we remove that section, run `charmcraft pack`, and then attempt to deploy the charm, the command will fail with a +```bash +Processing error: Failed to copy '/root/stage/src': no such file or directory. +``` + +### The plan + +````{dropdown} Detour: exploration of a lazy approach which turns out to be a bad idea. + +The minimal-effort solution in this case could be to create a file `/src/charm.py` and translate the built-in `self.on` event hooks to subprocess calls to the Bash event hooks as-they-are. Roughly: + +```python +#!/usr/bin/env python3 +import os +import ops + +class Microsample(ops.CharmBase): + def __init__(self, *args): + super().__init__(*args) + self.framework.observe(self.on.config_changed, lambda _: os.popen('../hooks/config-changed')) + self.framework.observe(self.on.install, lambda _: os.popen('../hooks/install')) + self.framework.observe(self.on.start, lambda _: os.popen('../hooks/start')) + self.framework.observe(self.on.stop, lambda _: os.popen('../hooks/stop')) + # etc... + +if __name__ == "__main__": + main(ops.Microsample) + +``` +This is obviously horrible, but we can verify that it actually works and it is a useful exercise to verify that the only difference between the hooks-based charm and this Ops charm is the syntax by which the developer has to map hook names to handler scripts. + +```{important} + + +We need a few preparatory steps:\ + • Add a `requirements.txt` file to ensure that the charm's Python environment will install for us the `ops` package.\ +• Modify the install hook to install `snap` for us, which is used in the script.\ +• In practice we cannot bind lambdas to `observe`, we need to write dedicated _methods_ for that.\ +• We need to figure out the required environment variables for the commands to work, which is not trivial.\ +\ +A more detailed explanation of this process is worthy of its own how-to guide, so we'll skip to the punchline here: it works. Check out [this branch](https://github.com/PietroPasotti/hooks-to-ops/tree/1-sh-charm) and see for yourself. + +``` + +```` + +It is in our interest to move the handler logic for each `/hooks/` to `Microsample._on_`, for several reasons: +- We can avoid code duplication by accessing shared data via the CharmBase interface provided through `self`. +- The code is all in one place, easier to maintain. +- We automatically have one Python object we can test, instead of going back and forth between Bash scripts and Python wrappers. +- We can use [the awesome testing Harness](https://juju.is/docs/sdk/testing). + +So let's do that. + +The idea is to turn those bash scripts into Python code we can call from aptly-named `Microsample` methods; but does it always make sense to do so? We'll see in a minute. + +### Step 1: Move script contents as-they-are into dedicated charm methods + + +```python +class Microsample(ops.CharmBase): + def __init__(self, framework): + super().__init__(framework) + framework.observe(self.on.install, self._on_install) + framework.observe(self.on.config_changed, self._on_config_changed) + framework.observe(self.on.start, self._on_start) + # etc ... +``` +Let's begin with `install`. +The `/hooks/install` script checks if a snap package is installed; if not, it installs it. We need to still reach out to a shell to grab the `snap` package info and install the package, but we can have the logic and the status management in Python, which is nice. We use `subprocessing.check_call` to reach out to the OS. And yes, there is a better way to do this, we'll get to that later. + +```python + def _on_install(self, _event): + snapinfo_cmd = Popen("snap info microsample".split(" "), + stdout=subprocess.PIPE) + output = check_output("grep -c 'installed'".split(" "), + stdin=snapinfo_cmd.stdout) + is_microsample_installed = bool(output.decode("ascii").strip()) + + if not is_microsample_installed: + self.unit.status = ops.MaintenanceStatus("installing microsample") + out = check_call("snap install microsample --edge") + + self.unit.status = ops.ActiveStatus() +``` + +For `on-start` and `on-stop`, which are simple instructions to `systemctl` to start/stop the `microsample` service, we can copy over the commands as they are: + +```python + def _on_start(self, _event): # noqa + check_call("systemctl start snap.microsample.microsample.service".split(' ')) + + def _on_stop(self, _event): # noqa + check_call("systemctl stop snap.microsample.microsample.service".split(' ')) +``` +In a couple of places in the scripts, `sleep 3` calls ensure that the service has some time to come up; however, this might get the charm stuck in the waiting loop if for whatever reason the service does NOT come up, so it is quite risky and we are not going to do that. Instead, we are going to rely on the fact that if other event handlers were to fail because of the service not being up, they would handle that case appropriately (e.g., defer the event if necessary). + +The rest of the translation is pretty straightforward. However, it is still useful to note a few things about relations, logging, and environment variables, which we do below. + +#### Wrapping the `website` relation + +`ops.model.Relation` provides a neat wrapper for the juju relation object. +We are going to add a helper method: + +```python + def _get_website_relation(self) -> ops.model.Relation: + # WARNING: would return None if called too early, e.g. during install + return self.model.get_relation("website") +``` + +That allows us to fetch the Relation wherever we need it and access its contents or mutate them in a natural way: +```python + def _on_website_relation_joined(self, _event): + relation = self._get_website_relation() + relation.data[self.unit].update( + {"hostname": self.private_address, + "port": self.port} + ) +``` + +Note how `relation.data` provides an interface to the relation databag (more on that [here](https://juju.is/docs/sdk/relations#heading--relation-data)) and we need to select which part of that bag to access by passing an `ops.model.Unit` instance. + +#### Logging + +Every maintainable charm will have some form of logging integrated; in a few places in the Bash scripts we see calls to a `juju-log` command; we can replace them with simple `logger.log` calls; such as in +```python + def _on_website_relation_departed(self, _event): # noqa + logger.debug("%s departed website relation", self.unit.name) +``` +Where `logger = logging.getLogger(__name__)`. + +#### Environment variables + +Some of the Bash scripts read environment variables such as `$JUJU_REMOTE_UNIT`, `$JUJU_UNIT_NAME` ; of course we could do + +```python +JUJU_UNIT_NAME = os.environ["JUJU_UNIT_NAME"] +``` + +but `CharmBase` exposes a `.unit` attribute we can read this information from, instead of grabbing it off the environment; this makes for more readable code. +So, wherever we need the juju unit name, we can write `self.unit.name` (that will get you `microsample/0` for example) or if you are actually after the *application name*, you can write `self.unit.app.name` (and get `microsample` back, without the unit index suffix). + +The resulting code at this stage can be inspected at [this branch](https://github.com/PietroPasotti/hooks-to-ops/tree/2-py-charm). + +### Step 2: Clean up snap & systemd code + +In the `_on_install` method we had translated one-to-one the calls to `snap info` to check whether the snap was installed or not; we can however use one more Linux lib for that: + +`charmcraft fetch-lib charms.operator_libs_linux.v1.snap` + +Then we can replace all that `Popen` piping with simpler calls into the lib's API; `_on_install `becomes: +```python + def _on_install(self, _event): + microsample_snap = snap.SnapCache()["microsample"] + if not microsample_snap.present: + self.unit.status = ops.MaintenanceStatus("installing microsample") + microsample_snap.ensure(snap.SnapState.Latest, channel="edge") + + self.wait_service_active() + self.unit.status = ops.ActiveStatus() + +``` + +Similarly all that string parsing we were doing to get a hold of the snap version, can be simplified by grabbing the `microsample_snap.channel` (not quite the same, but for the purposes of this charm, it is close enough). + +```python + def _get_microsample_version(self): + microsample_snap = snap.SnapCache()["microsample"] + return microsample_snap.channel +``` + +Also, we can interact with the microsample service via the `operator_libs_linux.v0` charm library, which wraps `systemd` and allows us to write simply: + +```python + def _on_start(self, _event): # noqa + systemd.service_start("snap.microsample.microsample.service") + + def _on_stop(self, _event): # noqa + systemd.service_stop("snap.microsample.microsample.service") +``` + +```{note} + +To install:\ +`charmcraft fetch-lib charms.operator_libs_linux.v0.systemd`\ +`charmcraft fetch-lib charms.operator_libs_linux.v1.snap`\ + +To use, add line to imports: +`from charms.operator_libs_linux.v0 import systemd` \ +`from charms.operator_libs_linux.v1 import snap` + +``` + +By inspecting more closely the flow of the events, we realize that not all of the event handlers that we currently subscribe to are necessary. For example, the relation data is going to be set once the relation is joined, but nothing needs to be done when the relation changes or is broken/departed. Since it depends on configurable values, however, we will need to make sure that the `config-changed` handler also keeps the relation data up to date. + +Furthermore we can get rid of the `start` handler, since the `snap.ensure()` call will also start the service for us on `install`. Similarly we can strip away most of the calls in `_on_upgrade_charm` (originally invoking multiple other hooks) and only call `snap.ensure(...)`. + +The final result can be inspected at [this branch](https://github.com/PietroPasotti/hooks-to-ops/tree/3-py-final). + + +## Closing notes + +We have seen how to turn a hooks-based charm to one using the state-of-the-art Ops framework---this basically boils down to moving code from files in a folder to methods in a `CharmBase` subclass. +That is, this is what it amounts to _to a developer_. But what about _the system_? + +The fact is, hooks charms can be written in Assembly, or any other language, so long as the shebang references something (a command / an interpreter) known to the script runner. The starting charm was as a result very lightweight, since it is written in Bash and that is included in the base Linux image. + +Ops charms, on the other hand, are Python charms. As such, even though Ops is not especially large in and of itself, Ops charms bring a virtual environment with them. That makes the resulting charm package somewhat heavier. That might be a consideration when the charm target is a resource-constrained system. + +### Todo's / disclaimers + + +Above, the `website` relation has not been tested; implementing the `Requires` part of it is also left as an exercise to the reader. diff --git a/docs/howto/write-integration-tests-for-a-charm.md b/docs/howto/write-integration-tests-for-a-charm.md new file mode 100644 index 000000000..c4d76d7d1 --- /dev/null +++ b/docs/howto/write-integration-tests-for-a-charm.md @@ -0,0 +1,399 @@ +(write-integration-tests-for-a-charm)= +# How to write integration tests for a charm + +> See also: {ref}`testing` + +This document shows how to write integration tests for a charm. + +```{important} + +Integration testing is only one part of a comprehensive testing strategy. See {ref}`How to test a charm ` for unit testing and {ref}`How to write a functional test ` for functional tests. + +``` + +The instructions all use the Juju `python-libjuju` client, either through the `pytest-operator` library or directly. + +> See more: [`python-libjuju`](https://github.com/charmed-kubernetes/pytest-operator), {ref}`pytest-operator` + +## Prepare your environment + +In order to run integrations tests you will need to have your environment set up with `tox` installed. + +> See more: {ref}`How to set up your development environment ` + +## Prepare the `tox.ini` configuration file + +Check that the next information is in your `tox.ini` file. If you initialised the charm with `charmcraft init` it should already be there. + +``` +[testenv:integration] +description = Run integration tests +deps = + pytest + juju + pytest-operator + -r {tox_root}/requirements.txt +commands = + pytest -v \ + -s \ + --tb native \ + --log-cli-level=INFO \ + {posargs} \ + {[vars]tests_path}/integration +``` + +

Create a test file

 + +By convention, integration tests are kept in the charm’s source tree, in a directory called `tests/integration`. + +If you initialised the charm with `charmcraft init`, your charm directory should already contain a `tests/integration/test_charm.py` file. Otherwise, create this directory structure manually (the test file can be called whatever you wish) and, inside the `.py` file, import `pytest` and, from the `pytest_operator.plugin`, the `OpsTest` class provided by the `ops_test` fixture: + +``` +import pytest +from pytest_operator.plugin import OpsTest +``` + +The `ops_test` fixture is your entry point to the `pytest-operator` library, and the preferred way of interacting with Juju in integration tests. This fixture will create a model for each test file -- if you write two tests that should not share a model, make sure to place them in different files. + +

Build your tests

 + +```{note} + +Use `pytest` custom markers to toggle which types of tests are being run so you can skip the destructive parts and focus on the business logic tests. See more: [Discourse | Pasotti: Classify tests with pytest custom markers for quick integration testing iterations](https://discourse.charmhub.io/t/classify-tests-with-pytest-custom-markers-for-quick-integration-testing-iterations/14006). + +``` + +

Test build and deploy

 + +To build and deploy the current charm, in your integration test file, add the function below: +``` +@pytest.mark.skip_if_deployed +@pytest.mark.abort_on_fail +async def test_build_and_deploy(ops_test: OpsTest): + charm = await ops_test.build_charm(".") + app = await ops_test.model.deploy(charm) + + await ops_test.model.wait_for_idle(status="active", timeout=60) +``` + +Tests run sequentially in the order they are written in the file. It can be useful to put tests that build and deploy applications in the top of the file as the applications can be used by other tests. For that reason, adding extra checks or `asserts` in this test is not recommended. + +The decorator `@pytest.mark.abort_on_fail` abort all next tests if something goes wrong. With the decorator `@pytest.mark.skip_if_deployed` you can skip that test if a `--model` is passed as a command line parameter (see [Run your tests](#heading--run-tests) for more information). + +`ops_test.build_charm` builds the charm with charmcraft. `ops_test.model` is an instance of `python-libjuju` 's [Model](https://pythonlibjuju.readthedocs.io/en/latest/api/juju.model.html#juju.model.Model) class that reference the active model tracked by `pytest-operator` for the current module. + +As an alternative to `wait_for_idle`, you can explicitly block until the application status is `active` or `error` and then assert that it is `active`. + +``` + await ops_test.model.block_until(lambda: app.status in ("active", "error"), timeout=60,) + assert app.status, "active" +``` + +> Example implementations: [charm-coredns](https://github.com/charmed-kubernetes/charm-coredns/blob/b1d83b6a31200924fefcd288336bc1f9323c6a72/tests/integration/test_integration.py#L21), [charm-calico](https://github.com/charmed-kubernetes/charm-calico/blob/e1dfdda92fefdba90b7b7e5247fbc861c34ad146/tests/integration/test_calico_integration.py#L18) + +> See more: +> - [`pytest-operator` | `ops_test.build_charm`](https://github.com/charmed-kubernetes/pytest-operator/blob/ab50fc20320d3ea3d8a37495f92a004531a4023f/pytest_operator/plugin.py#L1020) +> - [`python-libjuju` | `model.deploy `](https://github.com/juju/python-libjuju/blob/2581b0ced1df6201c6b7fd8cc0b20dcfa9d97c51/juju/model.py#L1658) + +

Deploy your charm with resources

 + +> See also: {ref}`Resource <12734md>` + +A charm can require `file` or `oci-image` `resources` to work, that can be provided to `ops_test.model.deploy`. In Charmhub, resources have revision numbers. For file resources already stored in Charmhub, you can use `ops_test.download_resources`: + +``` +async def test_build_and_deploy(ops_test: OpsTest): + charm = await ops_test.build_charm(".") + arch_resources = ops_test.arch_specific_resources(charm) + resources = await ops_test.download_resources(charm, resources=arch_resources) + app = await ops_test.model.deploy(charm, resources=resources) + await ops_test.model.wait_for_idle(status="active", timeout=60) +``` + +You can also reference a file resource on the filesystem. You can also use [`ops_test.build_resources`](https://github.com/charmed-kubernetes/pytest-operator/blob/ab50fc20320d3ea3d8a37495f92a004531a4023f/pytest_operator/plugin.py#L1073) to build file resources from a build script. + +For `oci-images` you can reference an image registry. +``` + ... + resources = {"resource_name": "localhost:32000/image_name:latest"} + app = await ops_test.model.deploy(charm, resources=resources) + ... +``` + +> Example implementations: [kubernetes-control-plane](https://github.com/charmed-kubernetes/charm-kubernetes-control-plane/blob/8769db394bf377a03ce94066307ecf831b88ad17/tests/integration/test_kubernetes_control_plane_integration.py#L41), [synapse-operator](https://github.com/canonical/synapse-operator/blob/eb44f4959a00040f08b98470f8b17cae4cc616da/tests/integration/conftest.py#L119), [prometheus-k8s](https://github.com/canonical/prometheus-k8s-operator/blob/d29f323343a1e4906a8c71104fcd1de817b2c2e6/tests/integration/test_remote_write_with_zinc.py#L27) + +> +> See more: +> - [`pytest-operator` | `build_resources`](https://github.com/charmed-kubernetes/pytest-operator/blob/ab50fc20320d3ea3d8a37495f92a004531a4023f/pytest_operator/plugin.py#L1073) +> - [`pytest-operator` | `download_resources`](https://github.com/charmed-kubernetes/pytest-operator/blob/ab50fc20320d3ea3d8a37495f92a004531a4023f/pytest_operator/plugin.py#L1101) +> - [`python-libjuju` | `model.deploy`](https://github.com/juju/python-libjuju/blob/2581b0ced1df6201c6b7fd8cc0b20dcfa9d97c51/juju/model.py#L1658) + + +

Test a relation

 + +To test an integration between two applications, you can just integrate them through +the model. Both applications have to be deployed beforehand. + +``` + ... +async def test_my_integration(ops_test: OpsTest): + # both application_1 and application_2 have to be deployed + # in the current test or a previous one. + await ops_test.model.integrate("application_1:relation_name_1", "application_2:relation_name_2") + await ops_test.model.wait_for_idle(status="active", timeout=60) + # check any assertion here + .... +``` + +> Example implementations: [slurmd-operator](https://github.com/canonical/slurmd-operator/blob/ffb24b05bec1b10cc512c060a4739358bfea0df0/tests/integration/test_charm.py#L89) + +> See more: [`python-libjuju` | `model.integrate`](https://github.com/juju/python-libjuju/blob/2581b0ced1df6201c6b7fd8cc0b20dcfa9d97c51/juju/model.py#L1476) + +

Test a configuration

 + +> See also: {ref}`Configuration <12734md>` + +You can set a configuration option in your application and check its results. + +``` +async def test_config_changed(ops_test: OpsTest): + ... + await ops_test.model.applications["synapse"].set_config({"server_name": "invalid_name"}) + # In this case, when setting server_name to "invalid_name" + # we could for example expect a blocked status. + await ops_test.model.wait_for_idle(status="blocked", timeout=60) + .... +``` +> See also: https://discourse.charmhub.io/t/how-to-add-a-configuration-option-to-a-charm/4458 +> +> See also: [python-libjuju | application.set_config](https://github.com/juju/python-libjuju/blob/2581b0ced1df6201c6b7fd8cc0b20dcfa9d97c51/juju/application.py#L591) + + + +

Test an action

 + +> See also: {ref}`Action <12734md>` + +You can execute an action on a unit and get its results. + +``` +async def test_run_action(ops_test: OpsTest): + action_register_user = await ops_test.model.applications["myapp"].units[0].run_action("register-user", username="ubuntu") + await action_register_user.wait() + assert action_register_user.status == "completed" + password = action_register_user.results["user-password"] + # We could for example check here that we can login with the new user +``` + +> See also: [python-libjuju | unit.run_action](https://github.com/juju/python-libjuju/blob/2581b0ced1df6201c6b7fd8cc0b20dcfa9d97c51/juju/unit.py#L274) + +

Interact with the workload

 + +To interact with the workload, you need to have access to it. This is dependent on many aspects of your application, environment and network topology. + +You can get information from your application or unit addresses using `await ops_test.model.get_status`. That way, if your application exposes a public address you can reference it. You can also try to connect to a unit address or public address. + +``` +async def test_workload_connectivity(ops_test: OpsTest): + status = await ops_test.model.get_status() + address = status.applications['my_app'].public_address + # Or you can try to connect to a concrete unit + # address = status.applications['my_app'].units['my_app/0'].public_address + # address = status.applications['my_app'].units['my_app/0'].address + appurl = f"http://{address}/" + r = requests.get(appurl) + assert r.status_code == 200 +``` + +How you can connect to a private or public address is dependent on your configuration, so you may need a different approach. + +> Example implementations: [mongodb-k8s-operator](https://github.com/canonical/mongodb-k8s-operator/blob/8b9ebbee3f225ca98175c25781f1936dc4a62a7d/tests/integration/metrics_tests/test_metrics.py#L33), [tempo-k8s-operator](https://github.com/canonical/tempo-k8s-operator/blob/78a1143d99af99a1a56fe9ff82b1a3563e4fd2f7/tests/integration/test_integration.py#L69), [synapse](https://github.com/canonical/synapse-operator/blob/eb44f4959a00040f08b98470f8b17cae4cc616da/tests/integration/conftest.py#L170) + +> See more: +> - {ref}`Charm development best practices > Fetching network information <12734md>` +> - {ref}``juju` CLI commands > juju expose <12734md>` + +

Run a subprocess command within Juju context

 + +You can run a command within the Juju context with: + +``` + ... + command = ["microk8s", "version"] + returncode, stdout, stderr = await ops_test.run(*command, check=True) + ... +``` + +You can similarly invoke the Juju CLI. This can be useful for cases where `python-libjuju` sees things differently than the Juju CLI. By default the environment variable `JUJU_MODEL` is set, +so you don't need to include the `-m` parameter. + +``` + .... + command = ["secrets"] + returncode, stdout, stderr = await ops_test.juju(*command, check=True) + .... +``` + +> Example implementations: [prometheus-k8s-operator](https://github.com/canonical/prometheus-k8s-operator/blob/d29f323343a1e4906a8c71104fcd1de817b2c2e6/tests/integration/conftest.py#L86), [hardware-observer-operator](https://github.com/canonical/hardware-observer-operator/blob/08c50798ca1c133a5d8ba5c889e0bcb09771300b/tests/functional/conftest.py#L14) + + +> See more: +> - [`pytest-operator` | `run`](https://github.com/charmed-kubernetes/pytest-operator/blob/ab50fc20320d3ea3d8a37495f92a004531a4023f/pytest_operator/plugin.py#L576) +> - [`pytest-operator` | `juju`](https://github.com/charmed-kubernetes/pytest-operator/blob/ab50fc20320d3ea3d8a37495f92a004531a4023f/pytest_operator/plugin.py#L624) + +

Use several models

 + +You can use `pytest-operator` with several models, in the same cloud or in +different clouds. This way you can, for example, integrate machine charms +with Kubernetes charms easily. + +You can track a new model with: + +``` + new_model = await ops_test.track_model("model_alias", + cloud_name="cloud_name", + credential_name="credentials") +``` + +`track_model` will track a model with alias `model_alias` (not the real model name). It maybe necessary to use `credential_name` if you do not use the same cloud that the controller. + +Using the new alias, you can switch context to the new created model, similar to `juju switch` command: + +``` + with ops_test.model_context("model_alias"): + # Here ops_test.model relates to the model referred by + # You can now use ops_test.model and it will apply to the model in the context +``` + +`pytest-operator` will handle the new created model by default. If you want to, you can remove it from the controller at any point: + +``` + await ops_test.forget_model("model_alias") +``` + +> Example implementations: [`charm-kubernetes-autoscaler`](https://github.com/charmed-kubernetes/charm-kubernetes-autoscaler/blob/8f4ddf5d66802ade73ed3aab2bb8d09fd9e4d63a/tests/integration/test_kubernetes_autoscaler.py#L31) + +> See more: +> - {ref}`Juju offers <12734md>` +> - {ref}`How to manage clouds <12734md>` +> - [pytest-operator | track_model](https://github.com/charmed-kubernetes/pytest-operator/blob/ab50fc20320d3ea3d8a37495f92a004531a4023f/pytest_operator/plugin.py#L720) +> - [pytest-operator | model_context](https://github.com/charmed-kubernetes/pytest-operator/blob/ab50fc20320d3ea3d8a37495f92a004531a4023f/pytest_operator/plugin.py#L480) +> - [pytest-operator | forget_model](https://github.com/charmed-kubernetes/pytest-operator/blob/ab50fc20320d3ea3d8a37495f92a004531a4023f/pytest_operator/plugin.py#L812) + + + +

Deploy a bundle

 + +```{note} + +It is not recommended to use `ops_test.build_bundle` and `ops_test.deploy_bundle` until this [issue](https://github.com/charmed-kubernetes/pytest-operator/issues/98) is closed, as it uses `juju-bundle` which is outdated. You can deploy bundles using `ops_test.model.deploy` or {ref}``ops_test.juju` <12734md>`. + +``` + + +

Render bundles and charms

 + +`pytest-operator` has utilities to template your charms and bundles using Jinja2. + +To render a kubernetes bundle with your current charm, create the file `./test/integration/bundle.yaml.j2` with this content: +``` +bundle: kubernetes +applications: + my-app: + charm: {{ charm }} + scale: {{ scale }} +``` + +You can now add the next integration test that will build an deploy the bundle with the current charm: +``` +async def test_build_and_deploy_bundle(ops_test: OpsTest): + charm = await ops_test.build_charm(".") + + bundle = ops_test.render_bundle( + 'tests/integration/bundle.yaml.j2', + charm=charm, + scale=1, + ) + juju_cmd = ["deploy", str(bundle)] + rc, stdout, stderr = await ops_test.juju(*juju_cmd) +``` + + +> Example implementations: [`hardware-observer-operator`](https://github.com/canonical/hardware-observer-operator/blob/47a79eb2872f6222099e7f48b8daafe8d20aa946/tests/functional/test_charm.py#L57) + + + +

Speed up `update_status` with `fast_forward`

 + +If your charm code depends on the `update_status` event, you can speed up its +firing rate with `fast_forward`. Inside the new async context you can put any code that will benefit from the new refresh rate so your test may execute faster. + +``` + ... + app = await ops_test.model.deploy(charm) + + async with ops_test.fast_forward(): + await ops_test.model.wait_for_idle(status="active", timeout=120) + .... +``` + +> Example implementations [`postgresql-k8s-operator`](https://github.com/canonical/postgresql-k8s-operator/blob/69b2c138fa6b974883aa6d3d15a3315189d321d8/tests/integration/ha_tests/test_upgrade.py#L58), [`synapse-operator`](https://github.com/canonical/synapse-operator/blob/05c00bb7666197d04f1c025c36d8339b10b64a1a/tests/integration/test_charm.py#L249) + + +> See more: +> - {ref}`Event `update-status` ` +> - [`pytest-operator` | `fast_forward`](https://github.com/charmed-kubernetes/pytest-operator/blob/ab50fc20320d3ea3d8a37495f92a004531a4023f/pytest_operator/plugin.py#L1400) + + +

Run your tests

 + +By default you can run all your tests with: + +``` +tox -e integration +``` + +These tests will use the context of the current controller in Juju, and by default will create a new model per module, that will be destroyed when the test is finished. The cloud, controller and model name can be specified with the parameters `--cloud`, `--controller` and `--model` parameters. + +If you specify the model name and do not delete the model on test tear down with the parameter `--keep-models`, you can reuse a model from a previous test run, as in the next example: +``` +# in the initial execution, the new model will be created +tox -e integration -- --keep-models --model test-example-model +# in the next execution it will reuse the model created previously: +tox -e integration -- --keep-models --model test-example-model --no-deploy +``` + +The parameter `--no-deploy` will skip tests decorated with `@pytest.mark.skip_if_deployed`. That way you can iterate faster on integration tests, as applications can be deployed only once. + +There are different ways of specifying a subset of tests to run using `pytest`. With the `-k` option you can specify different expressions. For example, the next command will run all tests in the `test_charm.py` file except `test_one` function. +``` +tox -e integration -- tests/integration/test_charm.py -k "not test_one" +``` + +> Example implementations: {ref}``mysql-k8s-operator` <12734md>` + +> See more: +> - [`pytest-operator` | `skip_if_deployed`](https://github.com/charmed-kubernetes/pytest-operator/blob/ab50fc20320d3ea3d8a37495f92a004531a4023f/pytest_operator/plugin.py#L139) +> - [`pytest | How to invoke pytest`](https://docs.pytest.org/en/7.1.x/how-to/usage.html) + +

Generate crash dumps

 + +To generate crash dumps, you need the `juju-crashdump` tool . + + +You can install it with `sudo snap install --classic juju-crashdump`. + +By default, when tests are run, a crash dump file will be created in the current directory if a test fails and if `--keep-models` is `false`. This crash dump file will include the current configuration and also Juju logs. + +You can disable crash dump generation with `--crash-dump=never`. To always create a crash dump file (even when tests do not fail) to a specific location run: + +``` +tox -e integration -- --crash-dump=always --crash-dump-output=/tmp +``` + +> See more: +> - [`juju-crashdump`](https://github.com/juju/juju-crashdump) +> - [`pytest-operator` | `--crash-dump`](https://github.com/charmed-kubernetes/pytest-operator/blob/ab50fc20320d3ea3d8a37495f92a004531a4023f/pytest_operator/plugin.py#L97) + + +> Contributors: @natalia-nowakowska , @javierdelapuente diff --git a/docs/howto/write-scenario-tests-for-a-charm.md b/docs/howto/write-scenario-tests-for-a-charm.md new file mode 100644 index 000000000..4cbcf7e89 --- /dev/null +++ b/docs/howto/write-scenario-tests-for-a-charm.md @@ -0,0 +1,81 @@ +(write-scenario-tests-for-a-charm)= +# How to write scenario tests for a charm + +First of all, install scenario: + +`pip install ops-scenario` + +Then, open a new `test_foo.py` file where you will put the test code. + +```python +# import the necessary objects from scenario and ops +from scenario import State, Context +import ops +``` + + +Then declare a new charm type: +```python +class MyCharm(ops.CharmBase): + pass +``` +And finally we can write a test function. The test code should use a Context object to encapsulate the charm type being tested (`MyCharm`) and any necessary metadata, then declare the initial `State` the charm will be presented when run, and `run` the context with an `event` and that initial state as parameters. +In code: + +```python +def test_charm_runs(): + # arrange: + # create a Context to specify what code we will be running + ctx = Context(MyCharm, meta={'name': 'my-charm'}) + # and create a State to specify what simulated data the charm being run will access + state_in = State() + # act: + # ask the context to run an event, e.g. 'start', with the state we have previously created + state_out = ctx.run(ctx.on.start(), state_in) + # assert: + # verify that the output state looks like you expect it to + assert state_out.status.unit.name == 'unknown' +``` + +> See more: +> - {ref}`State ` +> - {ref}`Context ` +> - {ref}`Event ` + +```{note} + +If you like using unittest, you should rewrite this as a method of some TestCase subclass. + +``` + +## Mocking beyond the State + +If you wish to use Scenario to test an existing charm type, you will probably need to mock out certain calls that are not covered by the {ref}`State ` data structure. +In that case, you will have to manually mock, patch or otherwise simulate those calls on top of what Scenario does for you. + +For example, suppose that the charm we're testing uses the `KubernetesServicePatch`. To update the test above to mock that object, modify the test file to contain: + +```python +import pytest +from unittest import patch + +@pytest.fixture +def my_charm(): + with patch("charm.KubernetesServicePatch"): + yield MyCharm +``` + +Then you should rewrite the test to pass the patched charm type to the Context, instead of the unpatched one. In code: +```python +def test_charm_runs(my_charm): + # arrange: + # create a Context to specify what code we will be running + ctx = Context(my_charm, meta={'name': 'my-charm'}) + # ... +``` + +```{note} + +If you use pytest, you should put the `my_charm` fixture in a toplevel `conftest.py`, as it will likely be shared between all your scenario tests. + +``` diff --git a/docs/howto/write-unit-tests-for-a-charm.md b/docs/howto/write-unit-tests-for-a-charm.md new file mode 100644 index 000000000..02b8cb54b --- /dev/null +++ b/docs/howto/write-unit-tests-for-a-charm.md @@ -0,0 +1,195 @@ +(write-unit-tests-for-a-charm)= +# How to write unit tests for a charm + +The Ops library provides a testing harness, so you can check your charm does the right thing in different scenarios without having to create a full deployment. When you run `charmcraft init`, the template charm it creates includes some sample tests, along with a `tox.ini` file; use `tox` to run the tests and to get a short report of unit test coverage. + +## Testing basics + +Here’s a minimal example, taken from the `charmcraft init` template with some additional comments: + +```python +# Import Ops library's testing harness +import ops +import ops.testing +import pytest +# Import your charm class +from charm import TestCharmCharm + + +@pytest.fixture +def harness(): + # Instantiate the Ops library's test harness + harness = ops.testing.Harness(TestCharmCharm) + # Set a name for the testing model created by Harness (optional). + # Cannot be called after harness.begin() + harness.set_model_name("testing") + # Instanciate an instance of the charm (harness.charm) + harness.begin() + yield harness + # Run Harness' cleanup method on teardown + harness.cleanup() + + +def test_config_changed(harness: ops.testing.Harness[TestCharmCharm]): + # Test initialisation of shared state in the charm + assert list(harness.charm._stored.things) == [] + + # Simulates the update of config, triggers a config-changed event + harness.update_config({"things": "foo"}) + # Test the config-changed method stored the update in state + assert list(harness.charm._stored.things) == ["foo"] + +``` + +We use [`pytest` unit testing framework](https://docs.pytest.org) (Python’s standard [unit testing framework](https://docs.python.org/3/library/unittest.html) is a valid alternative), augmenting it with [`Harness`](https://ops.readthedocs.io/en/latest/#ops.testing.Harness), the Ops library's testing harness. [`Harness`](https://ops.readthedocs.io/en/latest/#ops.testing.Harness) provides some convenient mechanisms for mocking charm events and processes. + +A common pattern is to specify some minimal `metadata.yaml` content for testing like this: + +```python +harness = Harness(TestCharmCharm, meta=''' + name: test-app + peers: + cluster: + interface: cluster + requires: + db: + interface: sql + ''') +harness.begin() +... +``` + +When using `Harness.begin()` you are responsible for manually triggering events yourself via other harness calls: + +```python +... +# Add a relation and trigger relation-created. +harness.add_relation('db', 'postgresql') # , +# Add a peer relation and trigger relation-created +harness.add_relation('cluster', 'test-app') # , +``` + +Notably, specifying relations in `charmcraft.yaml` does not automatically make them created by the +harness. If you have e.g. code that accesses relation data, you must manually add those relations +(including peer relations) for the harness to provide access to that relation data to your charm. + +In some cases it may be useful to start the test harness and fire the same hooks that Juju would fire on deployment. This can be achieved using the `begin_with_initial_hooks()` method , to be used in place of the `begin()` method. This method will trigger the events: `install -> relation-created -> config-changed -> start -> relation-joined` depending on whether any relations have been created prior calling `begin_with_initial_hooks()`. An example of this is shown in the [testing relations](https://juju.is/docs/sdk/relations) section. + +Using the `harness` variable, we can simulate various events in the charm’s lifecycle: + +```python +# Update the harness to set the active unit as a "leader" (the default value is False). +# This will trigger a leader-elected event +harness.set_leader(True) +# Update config. +harness.update_config({"foo": "bar", "baz": "qux"}) +# Disable hooks if we're about to do something that would otherwise cause a hook +# to fire such as changing configuration or setting a leader, but you don't want +# those hooks to fire. +harness.disable_hooks() +# Update config +harness.update_config({"foo": "quux"}) +# Re-enable hooks +harness.enable_hooks() +# Set the status of the active unit. We'd need "from ops.model import BlockedStatus". +harness.charm.unit.status = BlockedStatus("Testing") +``` + +Any of your charm’s properties and methods (including event callbacks) can be accessed using +`harness.charm`. You can check out the [harness API +docs](https://ops.readthedocs.io/en/latest/index.html#ops.testing.Harness) for more ways to use the +harness to trigger other events and to test your charm (e.g. triggering leadership-related events, +testing pebble events and sidecar container interactions, etc.). + + +## Testing log output + +Charm authors can also test for desired log output. Should a charm author create log messages in the standard form: + +```python +# ... +logger = logging.getLogger(__name__) + + +class SomeCharm(ops.CharmBase): +# ... + def _some_method(self): + logger.info("some message") +# ... +``` + +The above logging output could be tested like so: + +```python +# The caplog fixture is available in all pytest's tests +def test_logs(harness, caplog): + harness.charm._some_method() + with caplog.at_level(logging.INFO): + assert [rec.message for rec in caplog.records] == ["some message"] +``` + +## Simulating container networking + +> Added in 1.4, changed in version 2.0 + +In `ops` 1.4, functionality was added to the Harness to more accurately track connections to workload containers. As of `ops` 2.0, this behaviour is enabled and simulated by default (prior to 2.0, you had to enable it by setting `ops.testing.SIMULATE_CAN_CONNECT` to True before creating Harness instances). + +Containers normally start in a disconnected state, and any interaction with the remote container (push, pull, add_layer, and so on) will raise an `ops.pebble.ConnectionError`. + +To mark a container as connected, +you can either call [`harness.set_can_connect(container, True)`](https://ops.readthedocs.io/en/latest/#ops.testing.Harness.set_can_connect), or you can call [`harness.container_pebble_ready(container)`](https://ops.readthedocs.io/en/latest/#ops.testing.Harness.container_pebble_ready) if you want to mark the container as connected *and* trigger its pebble-ready event. + +However, if you're using [`harness.begin_with_initial_hooks()`](https://ops.readthedocs.io/en/latest/#ops.testing.Harness.begin_with_initial_hooks) in your tests, that will automatically call `container_pebble_ready()` for all containers in the charm's metadata, so you don't have to do it manually. + +If you have a hook that pushes a file to the container, like this: + +```python +def _config_changed(event): + c = self.unit.get_container('foo') + c.push(...) + self.config_value = ... +``` + +Your old testing code won't work: + +```python +@fixture +def harness(): + harness = Harness(ops.CharmBase, meta=""" + name: test-app + containers: + foo: + resource: foo-image + """) + harness.begin() + yield harness + harness.cleanup() + +def test_something(harness): + c = harness.model.unit.get_container('foo') + + # THIS NOW FAILS WITH A ConnectionError: + harness.update_config(key_values={'the-answer': 42}) +``` + +Which suggests that your `_config_changed` hook should probably use [`Container.can_connect()`](https://ops.readthedocs.io/en/latest/#ops.Container.can_connect): + +```python +def _config_changed(event): + c = self.unit.get_container('foo') + if not c.can_connect(): + # wait until we can connect + event.defer() + return + c.push(...) + self.config_value = ... +``` + +Now you can test both connection states: + +``` +harness.update_config(key_values={'the-answer': 42}) # can_connect is False +harness.container_pebble_ready('foo') # set can_connect to True +assert 42 == harness.charm.config_value +``` + diff --git a/docs/index.md b/docs/index.md new file mode 100644 index 000000000..3ad8b6fac --- /dev/null +++ b/docs/index.md @@ -0,0 +1,92 @@ +# Ops (`ops`) + +```{toctree} +:maxdepth: 2 +:hidden: true + +tutorial/index +howto/index +reference/index +explanation/index +``` + + + +The Ops (`ops`) library is a Python framework for writing and testing Juju charms. + +The library ([available on PyPI](https://pypi.org/project/ops/)) provides: + +- {ref}`ops_main_entry_point`, used to initialise and run your charm; +- {ref}`ops`, the API to respond to Juju events and manage the application; +- {ref}`ops_pebble`, the Pebble client, a low-level API for Kubernetes containers; +- {ref}`ops_testing`, the recommended API for unit testing charms; and +- {ref}`ops_testing_harness`, the legacy (deprecated API for unit testing charms. + +You can structure your charm however you like, but with the `ops` library, you get a framework that promotes consistency and readability by following best practices. It also helps you organise your code better by separating different aspects of the charm, such as managing the application's state, handling integrations with other services, and making the charm easier to test. + +If you're serious about charming, you *must* use `ops`! + +--------- + +## In this documentation + +````{grid} 1 1 2 2 + +```{grid-item-card} [Tutorial](tutorial/index) +:link: tutorial/index +:link-type: doc + +**Start here**: a hands-on introduction to Example Product for new users +``` + +```{grid-item-card} [How-to guides](/index) +:link: howto/index +:link-type: doc + +**Step-by-step guides** covering key operations and common tasks +``` + +```` + + +````{grid} 1 1 2 2 +:reverse: + +```{grid-item-card} [Reference](/index) +:link: reference/index +:link-type: doc + +**Technical information** - specifications, APIs, architecture +``` + +```{grid-item-card} [Explanation](/index) +:link: explanation/index +:link-type: doc + +**Discussion and clarification** of key topics +``` + +```` + + +--------- + + +## Project and community + +Ops is a member of the Ubuntu family. It’s an open source project that warmly welcomes community projects, contributions, suggestions, fixes and constructive feedback. + +* **[Read our code of conduct](https://ubuntu.com/community/ethos/code-of-conduct)**: +As a community we adhere to the Ubuntu code of conduct. + +* **[Get support](https://discourse.charmhub.io/)**: +Discourse is the go-to forum for all questions Ops. + +* **[Join our online chat](https://matrix.to/#/#charmhub-ops:ubuntu.com)**: +Meet us in the `#ops` channel on Matrix. + +* **[Report bugs](https://github.com/canonical/operator/)**: +We want to know about the problems so we can fix them. + +* **[Contribute docs](https://github.com/canonical/operator/tree/main/docs)**: +The documentation sources on GitHub. diff --git a/docs/index.rst b/docs/ops.rst similarity index 100% rename from docs/index.rst rename to docs/ops.rst diff --git a/docs/pebble.rst b/docs/pebble.rst deleted file mode 100644 index b5f6e8180..000000000 --- a/docs/pebble.rst +++ /dev/null @@ -1,6 +0,0 @@ -.. _ops_pebble_module: - -Pebble client -============= - -.. automodule:: ops.pebble diff --git a/docs/reference/api/index.md b/docs/reference/api/index.md new file mode 100644 index 000000000..4ec53d403 --- /dev/null +++ b/docs/reference/api/index.md @@ -0,0 +1,13 @@ +(ops-api-index)= +# `ops` API reference + +```{toctree} +:maxdepth: 2 + +ops-main-entrypoint +ops +ops_pebble +ops_testing +ops_testing_harness +``` + diff --git a/docs/reference/api/ops-main-entrypoint.rst b/docs/reference/api/ops-main-entrypoint.rst new file mode 100644 index 000000000..e4b1519ca --- /dev/null +++ b/docs/reference/api/ops-main-entrypoint.rst @@ -0,0 +1,16 @@ +.. _ops_main_entry_point: + +`ops.main` entry point +====================== + +The main entry point to initialise and run your charm. + +.. autofunction:: ops.main + +legacy main module +------------------ + +.. automodule:: ops.main + :noindex: + + diff --git a/docs/harness.rst b/docs/reference/api/ops-testing-harness.rst similarity index 95% rename from docs/harness.rst rename to docs/reference/api/ops-testing-harness.rst index a03cded49..b82c0c6d5 100644 --- a/docs/harness.rst +++ b/docs/reference/api/ops-testing-harness.rst @@ -1,6 +1,6 @@ -.. _harness: +.. _ops_testing_harness: -Harness (legacy unit testing) +`ops.testing.Harness` (legacy unit testing) ============================= .. deprecated:: 2.17 diff --git a/docs/state-transition-testing.rst b/docs/reference/api/ops-testing.rst similarity index 98% rename from docs/state-transition-testing.rst rename to docs/reference/api/ops-testing.rst index 156848023..76e4dba31 100644 --- a/docs/state-transition-testing.rst +++ b/docs/reference/api/ops-testing.rst @@ -1,6 +1,6 @@ -.. _state-transition-tests: +.. _ops_testing: -Unit testing (was: Scenario) +`ops.testing` (was: Scenario) ============================ Install ops with the ``testing`` extra to use this API; for example: diff --git a/docs/reference/api/ops.rst b/docs/reference/api/ops.rst new file mode 100644 index 000000000..91d44277d --- /dev/null +++ b/docs/reference/api/ops.rst @@ -0,0 +1,8 @@ +.. _ops: + +`ops` +===== + +.. automodule:: ops + :exclude-members: main + diff --git a/docs/reference/api/pebble.rst b/docs/reference/api/pebble.rst new file mode 100644 index 000000000..e826b591d --- /dev/null +++ b/docs/reference/api/pebble.rst @@ -0,0 +1,6 @@ +.. _ops_pebble: + +`ops.pebble` +============ + +.. automodule:: ops.pebble diff --git a/docs/reference/charm-relation-interfaces.md b/docs/reference/charm-relation-interfaces.md new file mode 100644 index 000000000..b6b5dfa82 --- /dev/null +++ b/docs/reference/charm-relation-interfaces.md @@ -0,0 +1,44 @@ +(charm-relation-interfaces)= +# Charm-relation-interfaces + +> See also: {ref}`manage-interfaces` + +[`charm-relation-interfaces`](https://github.com/canonical/charm-relation-interfaces) is a repository containing specifications, databag schemas and interface tests for Juju relation interfaces. In other words, it is the source of truth for data and behavior of providers and requirers of integrations. + +The purpose of this project is to provide uniformity in the landscape of all possible integrations and promote charm interoperability. + +Juju interfaces are untyped, which means that for juju to think two charms can be integrated all it looks at is whether the interface names of the two endpoints you're trying to connect are the same string. But it might be that the two charms have different, incompatible implementations of two different integrations that happen to have the same name. + +In order to prevent two separate charms from rolling their own integration with the same name, and prevent a sprawl of many subtly different interfaces with similar semantics and similar purposes, we introduced `charm-relation-interfaces`. + +## Using `charm-relation-interfaces` + +If you have a charm that provides a service, you should search `charm-relation-interfaces` (or directly charmhub in the future) and see if it exists already, or perhaps a similar one exists that lacks the semantics you need and can be extended to support it. + +Conversely, if the charm you are developing needs some service (a database, an ingress url, an authentication endpoint...) you should search `charm-relation-interfaces` to see if there is an interface you can use, and to find existing charms that provide it. + +There are three actors in play: + +* **the owner of the specification** of the interface, which also owns the tests that can be used to verify "does charm X 'really' support this interface?". This is the `charm-relation-interfaces` repo. +* **the owner of the implementation** of an interface. In practice, this often is the charm that owns the charm library with the reference implementation for an interface. +* **the interface user**: a charm that wants to use the interface (either as requirer or as provider). + +The interface user needs the implementation (typically, the provider also happens to be the owner and so it already has the implementation). This is addressed by `charmcraft fetch-lib`. + +The owner of the implementation needs the specification, to help check that the implementation is in fact compliant. + +## Repository structure + +For each interface, the charm-relation-interfaces repository hosts: +- the **specification**: a semi-formal definition of what the semantics of the interface is, and what its implementations are expected to do in terms of both the provider and the requirer +- a list of **reference charms**: these are the charms that implement this interface, typically, the owner of the charm library providing the original implementation. +- the **schema**: pydantic models unambiguously defining the accepted unit and application databag contents for provider and requirer. +- the **interface tests**: python tests that can be run to verify that a charm complies with the interface specification. + + +## Charm relation interfaces in Charmhub +In the future, Charmhub will have a searchable collection of integration interfaces. +Charmhub will, for all charms using the interface, verify that they implement it correctly (regardless of whether they use the 'official' implementation or they roll their own) in order to give the charm a happy checkmark on `charmhub.io`. In order to do that it will need to fetch the specification (from `charm-relation-interfaces`) *and* the charm repo, because we can't know what implementation they are using: we need the source code. + + + diff --git a/docs/reference/index.md b/docs/reference/index.md new file mode 100644 index 000000000..8fbcc6269 --- /dev/null +++ b/docs/reference/index.md @@ -0,0 +1,14 @@ +(reference)= +# Reference + +```{toctree} +:maxdepth: 2 + +charm-relation-interfaces +api/index +pytest-operator +testing +interface-tests + +``` + diff --git a/docs/reference/interface-tests.md b/docs/reference/interface-tests.md new file mode 100644 index 000000000..56580e288 --- /dev/null +++ b/docs/reference/interface-tests.md @@ -0,0 +1,39 @@ +(interface-tests)= +# Interface tests +> See also: {ref}`manage-interfaces` + +Interface tests are tests that verify the compliance of a charm with an interface specification. +Interface specifications, stored in {ref}`charm-relation-interfaces `, are contract definitions that mandate how a charm should behave when integrated with another charm over a registered interface. + +Interface tests will allow `charmhub` to validate the integrations of a charm and verify that your charm indeeed supports "the" `ingress` interface and not just an interface called "ingress", which happens to be the same name as "the official `ingress` interface v2" as registered in charm-relation-interfaces (see [here](https://github.com/canonical/charm-relation-interfaces/tree/main/interfaces/ingress/v2)). + +Also, they allow alternative implementations of an interface to validate themselves against the contractual specification stored in charm-relation-interfaces, and they help verify compliance with multiple versions of an interface. + +An interface test is a contract test powered by {ref}``Scenario` ` and a pytest plugin called [`pytest-interface-tester`](https://github.com/canonical/pytest-interface-tester). An interface test has the following pattern: +1) **GIVEN** an initial state of the relation over the interface under test +2) **WHEN** a specific relation event fires +3) **THEN** the state of the databags is valid (e.g. it satisfies an expected pydantic schema) + +On top of databag state validity, one can check for more elaborate conditions. + +A typical interface test will look like: + +```python +from interface_tester import Tester + +def test_data_published_on_changed_remote_valid(): + """This test verifies that if the remote end has published valid data and we receive a db-relation-changed event, then the schema is satisfied.""" + # GIVEN that we have a relation over "db" and the remote end has published valid data + relation = Relation(endpoint='db', interface='db', + remote_app_data={'model': '"bar"', 'port': '42', 'name': '"remote"', }, + remote_units_data={0: {'host': '"0.0.0.42"', }}) + t = Tester(State(relations=[relation])) + # WHEN the charm receives a db-relation-changed event + state_out = t.run(relation.changed_event) + # THEN the schema is valid + t.assert_schema_valid() +``` + +This allows us to, independently from what charm we are testing, determine if the behavioural specification of this interface is complied with. + + diff --git a/docs/reference/pytest-operator.md b/docs/reference/pytest-operator.md new file mode 100644 index 000000000..b86ec3767 --- /dev/null +++ b/docs/reference/pytest-operator.md @@ -0,0 +1,19 @@ +(pytest-operator)= +# `pytest-operator` + +`pytest-operator` is a Python library that provides Juju plugins for the generic Python library `pytest` to facilitate the {ref}`integration testing ` of charms. + +> See more: [`pytest-operator`](https://github.com/charmed-kubernetes/pytest-operator) + +It builds a fixture called `ops_test` that helps you interact with Juju through constructs that wrap around [`python-libjuju` ](https://pypi.org/project/juju/). + +> See more: +> - [`pytest-operator` > `ops_test`](https://github.com/charmed-kubernetes/pytest-operator/blob/main/docs/reference.md#ops_test) +> - [`pytest` > Fixtures](https://docs.pytest.org/en/6.2.x/fixture.html) + +It also provides convenient markers and command line parameters (e.g., the `@pytest.mark.skip_if_deployed` marker in combination with the `--no-deploy` configuration helps you skip, e.g., a deployment test in the case where you already have a deployment). + + +> See more: +> - [`pytest-operator` > Markers](https://github.com/charmed-kubernetes/pytest-operator/blob/main/docs/reference.md#markers) +> - [`pytest-operator` > Command line parameters](https://github.com/charmed-kubernetes/pytest-operator/blob/main/docs/reference.md#command-line-parameters) diff --git a/docs/reference/testing.md b/docs/reference/testing.md new file mode 100644 index 000000000..0ba3df7b1 --- /dev/null +++ b/docs/reference/testing.md @@ -0,0 +1,90 @@ +(testing)= +# Testing + +Charms should have tests to verify that they are functioning correctly. This document describes some of the various types of testing you may want to consider -- their meaning, recommended coverage, and recommended tooling in the context of a charm. + + + + +## Unit testing + +> See also: {ref}`write-unit-tests-for-a-charm`, {ref}`write-scenario-tests-for-a-charm` + +A **unit test** is a test that targets an individual unit of code (function, method, class, etc.) independently. In the context of a charm, it refers to testing charm code against mock Juju APIs and mocked-out workloads as a way to validate isolated behaviour without external interactions. + +Unit tests are intended to be isolating and fast to complete. These are the tests you would run every time before committing code changes. + +**Coverage.** Unit testing a charm should cover: + +- how relation data is modified as a result of an event +- what pebble services are running as a result of an event +- which configuration files are written and their contents, as a result of an event + +**Tools.** Unit testing a charm can be done using: + +- [`pytest`](https://pytest.org/) and/or [`unittest`](https://docs.python.org/3/library/unittest.html) and +- [`ops.testing.Harness`](https://operator-framework.readthedocs.io/en/latest/#module-ops.testing) and/or {ref}``ops-scenario` ` + + + + + +**Examples.** + +- [https://github.com/canonical/prometheus-k8s-operator/blob/main/tests/unit/test_charm.py](https://github.com/canonical/prometheus-k8s-operator/blob/main/tests/unit/test_charm.py) + +## Interface testing + +In the context of a charm, interface tests help validate charm library behavior without individual charm code against mock Juju APIs. + +> See more: {ref}`interface-tests` + + + +(integration-testing)= +## Integration testing +> See also: {ref}`write-integration-tests-for-a-charm` + +An **integration test** is a test that targets multiple software components in interaction. In the context of a charm, it checks that the charm operates as expected when Juju-deployed by a user in a test model in a real controller. + +Integration tests should be focused on a single charm. Sometimes an integration test requires multiple charms to be deployed for adequate testing, but ideally integration tests should not become end-to-end tests. + +Integration tests typically take significantly longer to run than unit tests. + +**Coverage.** + +* Charm actions +* Charm integrations +* Charm configurations +* That the workload is up and running, and responsive +* Upgrade sequence + * Regression test: upgrade stable/candidate/beta/edge from charmhub with the locally-built charm. + + +```{caution} + +When writing an integration test, it is not sufficient to simply check that Juju reports that running the action was successful; rather, additional checks need to be executed to ensure that whatever the action was intended to achieve worked. + +``` + +**Tools.** + +- [`pytest`](https://pytest.org/) and/or [`unittest`](https://docs.python.org/3/library/unittest.html) and +- [pytest-operator](https://github.com/charmed-kubernetes/pytest-operator) and/or [`zaza`](https://github.com/openstack-charmers/zaza) + + +**Examples.** + +- [https://github.com/canonical/prometheus-k8s-operator/blob/main/tests/integration/test_charm.py](https://github.com/canonical/prometheus-k8s-operator/blob/main/tests/integration/test_charm.py) + + diff --git a/docs/tutorial/from-zero-to-hero-write-your-first-kubernetes-charm/create-a-minimal-kubernetes-charm.md b/docs/tutorial/from-zero-to-hero-write-your-first-kubernetes-charm/create-a-minimal-kubernetes-charm.md new file mode 100644 index 000000000..c040ccecf --- /dev/null +++ b/docs/tutorial/from-zero-to-hero-write-your-first-kubernetes-charm/create-a-minimal-kubernetes-charm.md @@ -0,0 +1,435 @@ +(create-a-minimal-kubernetes-charm)= +# Create a minimal Kubernetes charm + +> {ref}`From Zero to Hero: Write your first Kubernetes charm ` > Create a minimal Kubernetes charm +> +> **See previous: {ref}`Set up your development environment `** + + + + + +As you already know from your knowledge of Juju, when you deploy a Kubernetes charm, the following things happen: + +1. The Juju controller provisions a pod with two containers, one for the Juju unit agent and the charm itself and one container for each application workload container that is specified in the `containers` field of a file in the charm that is called `charmcraft.yaml`. +1. The same Juju controller injects Pebble -- a lightweight, API-driven process supervisor -- into each workload container and overrides the container entrypoint so that Pebble starts when the container is ready. +1. When the Kubernetes API reports that a workload container is ready, the Juju controller informs the charm that the instance of Pebble in that container is ready. At that point, the charm knows that it can start communicating with Pebble. +1. Typically, at this point the charm will make calls to Pebble so that Pebble can configure and start the workload and begin operations. + +> Note: In the past, the containers were specified in a `metadata.yaml` file, but the modern practice is that all charm specification is in a single `charmcraft.yaml` file. + + + + + + + +All subsequent workload management happens in the same way -- the Juju controller sends events to the charm and the charm responds to these events by managing the workload application in various ways via Pebble. The picture below illustrates all of this for a simple case where there is just one workload container. + + +![Create a minimal Kubernetes charm](create_a_minimal_kubernetes_charm.png) + + +As a charm developer, your first job is to use this knowledge to create the basic structure and content for your charm: + + - descriptive files (e.g., YAML configuration files like the `charmcraft.yaml` file mentioned above) that give Juju, Python, or Charmcraft various bits of information about your charm, and +- executable files (like the `src/charm.py` file that we will see shortly) where you will use Ops-enriched Python to write all the logic of your charm. + + +## Set the basic information, requirements, and workload for your charm + +Create a file called `charmcraft.yaml`. This is a file that describes metadata such as the charm name, purpose, environment constraints, workload containers, etc., in short, all the information that tells Juju what it can do with your charm. + +In this file, do all of the following: + +First, add basic information about your charm: + +```text +name: demo-api-charm +title: | + demo-fastapi-k8s +description: | + This is a demo charm built on top of a small Python FastAPI server. + This charm could be related to PostgreSQL charm and COS Lite bundle (Canonical Observability Stack). +summary: | + FastAPI Demo charm for Kubernetes +``` + +Second, add an environment constraint assuming the latest major Juju version and a Kubernetes-type cloud: + +```text +assumes: + - juju >= 3.1 + - k8s-api +``` + +Third, describe the workload container, as below. Below, `demo-server` is the name of the container, and `demo-server-image` is the name of its OCI image. + +```text +containers: + demo-server: + resource: demo-server-image +``` + + +Fourth, describe the workload container resources, as below. The name of the resource below, `demo-server-image`, is the one you defined above. + +```text +resources: + # An OCI image resource for each container listed above. + # You may remove this if your charm will run without a workload sidecar container. + demo-server-image: + type: oci-image + description: OCI image from GitHub Container Repository + # The upstream-source field is ignored by Juju. It is included here as a reference + # so the integration testing suite knows which image to deploy during testing. This field + # is also used by the 'canonical/charming-actions' Github action for automated releasing. + upstream-source: ghcr.io/canonical/api_demo_server:1.0.1 +``` + + + +## Define the charm initialisation and application services + + + +Create a file called `requirements.txt`. This is a file that describes all the required external Python dependencies that will be used by your charm. + + +In this file, declare the `ops` dependency, as below. At this point you're ready to start using constructs from the Ops library. + +``` +ops >= 2.11 +``` + + +Create a file called `src/charm.py`. This is the file that you will use to write all the Python code that you want your charm to execute in response to events it receives from the Juju controller. + + +This file needs to be executable. One way you can do this is: + +```text +chmod a+x src/charm.py +``` + +In this file, do all of the following: + +First, add a shebang to ensure that the file is directly executable. Then, import the `ops` package to access the`CharmBase` class and the `main` function. Next, use `CharmBase` to create a charm class `FastAPIDemoCharm` and then invoke this class in the `main` function of Ops. As you can see, a charm is a pure Python class that inherits from the CharmBase class of Ops and which we pass to the `main` function defined in the `ops.main` module. + +```python +#!/usr/bin/env python3 + +import ops + +class FastAPIDemoCharm(ops.CharmBase): + """Charm the service.""" + + def __init__(self, framework: ops.Framework) -> None: + super().__init__(framework) + +if __name__ == "__main__": # pragma: nocover + ops.main(FastAPIDemoCharm) + +``` + + +Now, in the `__init__` function of your charm class, use Ops constructs to add an observer for when the Juju controller informs the charm that the Pebble in its workload container is up and running, as below. As you can see, the observer is a function that takes as an argument an event and an event handler. The event name is created automatically by Ops for each container on the template `-pebble-ready`. The event handler is a method in your charm class that will be executed when the event is fired; in this case, you will use it to tell Pebble how to start your application. + +```python +framework.observe(self.on.demo_server_pebble_ready, self._on_demo_server_pebble_ready) +``` + + +```{important} + +**Generally speaking:** A charm class is a collection of event handling methods. When you want to install, remove, upgrade, configure, etc., an application, Juju sends information to your charm. Ops translates this information into events and your job is to write event handlers + +``` + +```{tip} + +**Pro tip:** Use `__init__` to hold references (pointers) to other `Object`s or immutable state only. That is because a charm is reinitialised on every event. See {ref}`Talking to a workload: Control flow from A to Z `. + +``` + + + + +Next, define the event handler, as follows: + +We'll use the `ActiveStatus` class to set the charm status to active. Note that almost everything you need to define your charm is in the `ops` package that you imported earlier - there's no need to add additional imports. + +Use `ActiveStatus` as well as further Ops constructs to define the event handler, as below. As you can see, what is happening is that, from the `event` argument, you extract the workload container object in which you add a custom layer. Once the layer is set you replan your service and set the charm status to active. + + + + +```python +def _on_demo_server_pebble_ready(self, event: ops.PebbleReadyEvent) -> None: + """Define and start a workload using the Pebble API. + + Change this example to suit your needs. You'll need to specify the right entrypoint and + environment configuration for your specific workload. + + Learn more about interacting with Pebble at at https://juju.is/docs/sdk/pebble + Learn more about Pebble layers at + https://canonical-pebble.readthedocs-hosted.com/en/latest/reference/layers + """ + # Get a reference the container attribute on the PebbleReadyEvent + container = event.workload + # Add initial Pebble config layer using the Pebble API + container.add_layer("fastapi_demo", self._pebble_layer, combine=True) + # Make Pebble reevaluate its plan, ensuring any services are started if enabled. + container.replan() + # Learn more about statuses in the SDK docs: + # https://juju.is/docs/sdk/status + self.unit.status = ops.ActiveStatus() +``` + +The custom Pebble layer that you just added is defined in the `self._pebble_layer` property. Update this property to match your application, as follows: + +In the `__init__` method of your charm class, name your service to `fastapi-service` and add it as a class attribute : + +``` +self.pebble_service_name = "fastapi-service" +``` + +Finally, define the `pebble_layer` function as below. The `command` variable represents a command line that should be executed in order to start our application. + +```python +@property +def _pebble_layer(self) -> ops.pebble.Layer: + """A Pebble layer for the FastAPI demo services.""" + command = ' '.join( + {ref}` + 'uvicorn', + 'api_demo_server.app:app', + '--host=0.0.0.0', + '--port=8000', + ] + ) + pebble_layer: ops.pebble.LayerDict = { + 'summary': 'FastAPI demo service', + 'description': 'pebble config layer for FastAPI demo server', + 'services': { + self.pebble_service_name: { + 'override': 'replace', + 'summary': 'fastapi demo', + 'command': command, + 'startup': 'enabled', + } + }, + } + return ops.pebble.Layer(pebble_layer) +``` + + + +## Add logger functionality + +In the `src/charm.py` file, in the imports section, import the Python `logging` module and define a logger object, as below. This will allow you to read log data in `juju`. + +``` +import logging + +# Log messages can be retrieved using juju debug-log +logger = logging.getLogger(__name__) +``` + +## Tell Charmcraft how to build your charm + +In the same `charmcraft.yaml` file you created earlier, you need to describe all the information needed for Charmcraft to be able to pack your charm. In this file, do the following: + +First, add the block below. This will identify your charm as a charm (as opposed to something else you might know from using Juju, namely, a bundle). + +``` +type: charm +``` + + +Also add the block below. This declares that your charm will build and run charm on Ubuntu 22.04. + +``` +bases: + - build-on: + - name: ubuntu + channel: "22.04" + run-on: + - name: ubuntu + channel: "22.04" +``` + + +Aaaand that's it! Time to validate your charm! + +```{tip} + +Once you've mastered the basics, you can speed things up by navigating to your empty charm project directory and running `charmcraft init --profile kubernetes`. This will create all the files above and more, along with helpful descriptor keys and code scaffolding. + +``` + + +## Validate your charm + +First, ensure that you are inside the Multipass Ubuntu VM, in the `~/fastapi-demo` folder: + +``` +multipass shell charm-dev +cd ~/fastapi-demo +``` + +Now, pack your charm project directory into a `.charm` file, as below. This will produce a `.charm` file. In our case it was named `demo-api-charm_ubuntu-22.04-amd64.charm`; yours should be named similarly, though the name might vary slightly depending on your architecture. + +``` +charmcraft pack +# Packed demo-api-charm_ubuntu-22.04-amd64.charm +``` + +```{important} + +If packing failed - perhaps you forgot to make the charm.py executable earlier - you may need to run `charmcraft clean` before re-running `charmcraft pack`. `charmcraft` will generally detect when files have changed, but will miss only file attributes changing. + +``` + +```{important} + +**Did you know?** A `.charm` file is really just a zip file of your charm files and code dependencies that makes it more convenient to share, publish, and retrieve your charm contents. + +``` + + + + + + + +Deploy the `.charm` file, as below. Juju will create a Kubernetes `StatefulSet` named after your application with one replica. + +``` +juju deploy ./demo-api-charm_ubuntu-22.04-amd64.charm --resource \ + demo-server-image=ghcr.io/canonical/api_demo_server:1.0.1 +``` + + +```{important} + +**If you've never deployed a local charm (i.e., a charm from a location on your machine) before:**
As you may know, when you deploy a charm from Charmhub it is sufficient to run `juju deploy `. However, to deploy a local charm you need to explicitly define a `--resource` parameter with the same resource name and resource upstream source as in the `charmcraft.yaml`. + +``` + + +Monitor your deployment: + +``` +juju status --watch 1s +``` + +When all units are settled down, you should see the output below, where `10.152.183.215` is the IP of the K8s Service and `10.1.157.73` is the IP of the pod. + +``` +Model Controller Cloud/Region Version SLA Timestamp +welcome-k8s tutorial-controller microk8s/localhost 3.0.0 unsupported 13:38:19+01:00 + +App Version Status Scale Charm Channel Rev Address Exposed Message +demo-api-charm active 1 demo-api-charm 1 10.152.183.215 no + +Unit Workload Agent Address Ports Message +demo-api-charm/0* active idle 10.1.157.73 +``` + +Now, validate that the app is running and reachable by sending an HTTP request as below, where `10.1.157.73` is the IP of our pod and `8000` is the default application port. + +``` +curl 10.1.157.73:8000/version +``` + +You should see a JSON string with the version of the application: + +``` +{"version":"1.0.0"} +``` + +---- +```{dropdown} Expand if you wish to inspect your deployment further + + +1. Run: + ``` +kubectl get namespaces +``` +You should see that Juju has created a namespace called `welcome-k8s`. + +2. Try: + +``` +kubectl -n welcome-k8s get pods +``` + +You should see that your application has been deployed in a pod that has 2 containers running in it, one for the charm and one for the application. The containers talk to each other via the Pebble API using the UNIX socket. + +``` +NAME READY STATUS RESTARTS AGE +modeloperator-5df6588d89-ghxtz 1/1 Running 3 (7d2h ago) 13d +demo-api-charm-0 2/2 Running 0 7d2h +``` + +3. Check also: + +``` +kubectl -n welcome-k8s describe pod demo-api-charm-0 +``` +In the output you should see the definition for both containers. You'll be able to verify that the default command and arguments for our application container (`demo-server`) have been displaced by the Pebble service. You should be able to verify the same for the charm container (`charm`). + + +``` + +------------ + + +**Congratulations, you've successfully created a minimal Kubernetes charm!** + + +## Review the final code + +For the full code see: [01_create_minimal_charm](https://github.com/canonical/juju-sdk-tutorial-k8s/tree/01_create_minimal_charm) + +For a comparative view of the code before and after our edits see: +[Comparison](https://github.com/canonical/juju-sdk-tutorial-k8s/compare/main...01_create_minimal_charm) + + + +>**See next: {ref}`Make your charm configurable `** + diff --git a/docs/tutorial/from-zero-to-hero-write-your-first-kubernetes-charm/expose-operational-tasks-via-actions.md b/docs/tutorial/from-zero-to-hero-write-your-first-kubernetes-charm/expose-operational-tasks-via-actions.md new file mode 100644 index 000000000..313e64482 --- /dev/null +++ b/docs/tutorial/from-zero-to-hero-write-your-first-kubernetes-charm/expose-operational-tasks-via-actions.md @@ -0,0 +1,166 @@ +(expose-operational-tasks-via-actions)= +# Expose operational tasks via actions + +> {ref}`From Zero to Hero: Write your first Kubernetes charm ` > Expose operational tasks via actions +> +> **See previous: {ref}`Preserve your charm's data `** + + +````{important} + +This document is part of a series, and we recommend you follow it in sequence. However, you can also jump straight in by checking out the code from the previous branches: + + +```text +git clone https://github.com/canonical/juju-sdk-tutorial-k8s.git +cd juju-sdk-tutorial-k8s +git checkout 05_preserve_charm_data +git checkout -b 06_create_actions +``` + +```` + +A charm should ideally cover all the complex operational logic within the code, to help avoid the need for manual human intervention. + +Unfortunately, that is not always possible. As a charm developer, it is thus useful to know that you can also expose charm operational tasks to the charm user by defining special methods called `actions`. + +This can be done by adding an `actions` section in your `charmcraft.yaml` file and then adding action event handlers to the `src/charm.py` file. + +In this part of the tutorial we will follow this process to add an action that will allow a charm user to view the current database access points and, if set, also the username and the password. + + +## Define the actions + +Open the `charmcraft.yaml` file and add to it a block defining an action, as below. As you can see, the action is called `get-db-info` and it is intended to help the user access database authentication information. The action has a single parameter, `show-password`; if set to `True`, it will show the username and the password. + +```yaml +actions: + get-db-info: + description: Fetches Database authentication information + params: + show-password: + description: "Show username and password in output information" + type: boolean + default: False +``` + + +## Define the action event handlers + +Open the `src/charm.py` file. + +In the charm `__init__` method, add an action event observer, as below. As you can see, the name of the event consists of the name defined in the `charmcraft.yaml` file (`get-db-info`) and the word `action`. + +```python +# events on custom actions that are run via 'juju run-action' +framework.observe(self.on.get_db_info_action, self._on_get_db_info_action) +``` + + + + + +Now, define the action event handler, as below: First, read the value of the parameter defined in the `charmcraft.yaml` file (`show-password`). Then, use the `fetch_postgres_relation_data` method (that we defined in a previous chapter) to read the contents of the database relation data and, if the parameter value read earlier is `True`, add the username and password to the output. Finally, use `event.set_results` to attach the results to the event that has called the action; this will print the output to the terminal. + +If we are not able to get the data (for example, if the charm has not yet been integrated with the postgresql-k8s application) then we use the `fail` method of the event to let the user know. + + + +```python +def _on_get_db_info_action(self, event: ops.ActionEvent) -> None: + """This method is called when "get_db_info" action is called. It shows information about + database access points by calling the `fetch_postgres_relation_data` method and creates + an output dictionary containing the host, port, if show_password is True, then include + username, and password of the database. + If PSQL charm is not integrated, the output is set to "No database connected". + + Learn more about actions at https://juju.is/docs/sdk/actions + """ + show_password = event.params['show-password'] # see charmcraft.yaml + db_data = self.fetch_postgres_relation_data() + if not db_data: + event.fail('No database connected') + return + output = { + 'db-host': db_data.get('db_host', None), + 'db-port': db_data.get('db_port', None), + } + if show_password: + output.update( + { + 'db-username': db_data.get('db_username', None), + 'db-password': db_data.get('db_password', None), + } + ) + event.set_results(output) +``` + +## Validate your charm + +First, repack and refresh your charm: + +```text +charmcraft pack +juju refresh \ + --path="./demo-api-charm_ubuntu-22.04-amd64.charm" \ + demo-api-charm --force-units --resource \ + demo-server-image=ghcr.io/canonical/api_demo_server:1.0.1 +``` + + +Next, test that the basic action invocation works: + +```text +juju run demo-api-charm/0 get-db-info --wait 1m +``` + +It might take a few seconds, but soon you should see an output similar to the one below, showing the database host and port: + +```text +Running operation 13 with 1 task + - task 14 on unit-demo-api-charm-0 + +Waiting for task 14... +db-host: postgresql-k8s-primary.model2.svc.cluster.local +db-port: "5432" +``` + +Now, test that the action parameter (`show-password`) works as well by setting it to `True`: + +```text +juju run demo-api-charm/0 get-db-info show-password=True --wait 1m +``` + +The output should now include the username and the password: +``` +Running operation 15 with 1 task + - task 16 on unit-demo-api-charm-0 + +Waiting for task 16... +db-host: postgresql-k8s-primary.model2.svc.cluster.local +db-password: RGv80aF9WAJJtExn +db-port: "5432" +db-username: relation_id_4 +``` + + +Congratulations, you now know how to expose operational tasks via actions! + +## Review the final code + +For the full code see: [06_create_actions](https://github.com/canonical/juju-sdk-tutorial-k8s/tree/06_create_actions) + +For a comparative view of the code before and after this doc see: [Comparison](https://github.com/canonical/juju-sdk-tutorial-k8s/compare/05_preserve_charm_data...06_create_actions) + + +> **See next: {ref}`Observe your charm with COS Lite `** diff --git a/docs/tutorial/from-zero-to-hero-write-your-first-kubernetes-charm/expose-the-version-of-the-application-behind-your-charm.md b/docs/tutorial/from-zero-to-hero-write-your-first-kubernetes-charm/expose-the-version-of-the-application-behind-your-charm.md new file mode 100644 index 000000000..a8cc788a3 --- /dev/null +++ b/docs/tutorial/from-zero-to-hero-write-your-first-kubernetes-charm/expose-the-version-of-the-application-behind-your-charm.md @@ -0,0 +1,118 @@ +(expose-the-version-of-the-application-behind-your-charm)= +# Expose the version of the application behind your charm + +> {ref}`From Zero to Hero: Write your first Kubernetes charm ` > Expose the version of the application behind your charm +> +> **See previous: {ref}`Make your charm configurable `** + +````{important} + +This document is part of a series, and we recommend you follow it in sequence. However, you can also jump straight in by checking out the code from the previous branches: + +``` +git clone https://github.com/canonical/juju-sdk-tutorial-k8s.git +cd juju-sdk-tutorial-k8s +git checkout 02_make_your_charm_configurable +git checkout -b 03_set_workload_version +``` + +```` + +In this chapter of the tutorial you will learn how to expose the version of the application (workload) run by the charm -- something that a charm user might find it useful to know. + + +## Define functions to collect the workload application version and set it in the charm + +As a first step we need to add two helper functions that will send an HTTP request to our application to get its version. If the container is available, we can send a request using the `requests` Python library and then add class methods to parse the JSON output to get a version string, as shown below: + +- Import the `requests` Python library: + +```python +import requests +``` + +- Add the following class methods: + + +```python +@property +def version(self) -> str: + """Reports the current workload (FastAPI app) version.""" + try: + if self.container.get_services(self.pebble_service_name): + return self._request_version() + # Catching Exception is not ideal, but we don't care much for the error here, and just + # default to setting a blank version since there isn't much the admin can do! + except Exception as e: + logger.warning("unable to get version from API: %s", str(e), exc_info=True) + return "" + +def _request_version(self) -> str: + """Helper for fetching the version from the running workload using the API.""" + resp = requests.get(f"http://localhost:{self.config['server-port']}/version", timeout=10) + return resp.json()["version"] +``` + +Next, we need to update the `_update_layer_and_restart` method to set our workload version. Insert the following lines before setting `ActiveStatus`: + +```python +# Add workload version in Juju status. +self.unit.set_workload_version(self.version) +``` + +## Declare Python dependencies + + +Since we've added a third party Python dependency into our project, we need to list it in `requirements.txt`. Edit the file to add the following line: + +``` +requests~=2.28 +``` + +Next time you run `charmcraft` it will fetch this new dependency into the charm package. + + +## Validate your charm + +We've exposed the workload version behind our charm. Let's test that it's working! + +First, repack and refresh your charm: + +```text +charmcraft pack +juju refresh \ + --path="./demo-api-charm_ubuntu-22.04-amd64.charm" \ + demo-api-charm --force-units --resource \ + demo-server-image=ghcr.io/canonical/api_demo_server:1.0.1 +``` + +Our charm should fetch the application version and forward it to `juju`. Run `juju status` to check: + +```text +juju status +``` + +Indeed, the version of our workload is now displayed -- see the App block, the Version column: + +```text +Model Controller Cloud/Region Version SLA Timestamp +charm-model tutorial-controller microk8s/localhost 3.0.0 unsupported 12:37:27+01:00 + +App Version Status Scale Charm Channel Rev Address Exposed Message +demo-api-charm 1.0.1 active 1 demo-api-charm 0 10.152.183.233 no + +Unit Workload Agent Address Ports Message +demo-api-charm/0* active idle 10.1.157.75 +``` + +## Review the final code + + +For the full code see: [03_set_workload_version](https://github.com/canonical/juju-sdk-tutorial-k8s/tree/03_set_workload_version) + +For a comparative view of the code before and after this doc see: [Comparison](https://github.com/canonical/juju-sdk-tutorial-k8s/compare/02_make_your_charm_configurable...03_set_workload_version) + + +> **See next: {ref}`Integrate your charm with PostgreSQL `** + + diff --git a/docs/tutorial/from-zero-to-hero-write-your-first-kubernetes-charm/index.md b/docs/tutorial/from-zero-to-hero-write-your-first-kubernetes-charm/index.md new file mode 100644 index 000000000..623f1d785 --- /dev/null +++ b/docs/tutorial/from-zero-to-hero-write-your-first-kubernetes-charm/index.md @@ -0,0 +1,89 @@ +(from-zero-to-hero-write-your-first-kubernetes-charm)= +# From zero to hero: Write your first Kubernetes charm + +This tutorial will introduce you to the official way to write a Kubernetes charm --- that is, how to equip an application with all the operational logic that it needs so that you can manage it on any Kubernetes cloud with just a few commands, using [Juju](https://juju.is/docs/olm). + +```{important} + +**Did you know?** Writing a charm is also known as 'charming'! + +``` + + + + + +**What you'll need:** + +- A working station, e.g., a laptop with amd64 architecture. +- Familiarity with [Juju](https://juju.is/docs/juju) +- Familiarity with the Python programming language, Object-Oriented Programming, Event Handlers +- Understanding of Kubernetes fundamentals + + +**What you'll do:** + + + + +```{toctree} +:maxdepth: 1 + +set-up-your-development-environment +study-your-application +create-a-minimal-kubernetes-charm +make-your-charm-configurable +expose-the-version-of-the-application-behind-your-charm +integrate-your-charm-with-postgresql +preserve-your-charms-data +expose-operational-tasks-via-actions +observe-your-charm-with-cos-lite +write-unit-tests-for-your-charm +write-scenario-tests-for-your-charm +write-integration-tests-for-your-charm +open-a-kubernetes-port-in-your-charm +publish-your-charm-on-charmhub +``` + + + +(tutorial-kubernetes-next-steps)= +## Next steps + +By the end of this tutorial you will have built a machine charm and evolved it in a number of typical ways. But there is a lot more to explore: + +| If you are wondering... | visit... | +|-------------------------|----------------------| +| "How do I...?" | {ref}`howto-guides` | +| "What is...?" | {ref}`reference` | +| "Why...?", "So what?" | {ref}`explanation` | + + + + +
+ +> **Contributors:** @beliaev-maksim diff --git a/docs/tutorial/from-zero-to-hero-write-your-first-kubernetes-charm/integrate-your-charm-with-postgresql.md b/docs/tutorial/from-zero-to-hero-write-your-first-kubernetes-charm/integrate-your-charm-with-postgresql.md new file mode 100644 index 000000000..b83f70ffe --- /dev/null +++ b/docs/tutorial/from-zero-to-hero-write-your-first-kubernetes-charm/integrate-your-charm-with-postgresql.md @@ -0,0 +1,389 @@ +(integrate-your-charm-with-postgresql)= +# Integrate your charm with PostgreSQL + +> {ref}`From Zero to Hero: Write your first Kubernetes charm ` > Integrate your charm with PostgreSQL +> +> **See previous: {ref}`Expose the version of the application behind your charm `** + +````{important} + +This document is part of a series, and we recommend you follow it in sequence. However, you can also jump straight in by checking out the code from the previous branches: + + +```text +git clone https://github.com/canonical/juju-sdk-tutorial-k8s.git +cd juju-sdk-tutorial-k8s +git checkout 03_set_workload_version +git checkout -b 04_integrate_with_psql +``` + +```` + +A charm often requires or supports relations to other charms. For example, to make our application fully functional we need to connect it to the PostgreSQL database. In this chapter of the tutorial we will update our charm so that it can be integrated with the existing [PostgreSQL charm](https://charmhub.io/postgresql-k8s?channel=14/stable). + + + +## Fetch the required database interface charm libraries + +Navigate to your charm directory and fetch the [data_interfaces](https://charmhub.io/data-platform-libs/libraries/data_interfaces) charm library from Charmhub: + +``` +ubuntu@charm-dev:~/fastapi-demo$ charmcraft fetch-lib charms.data_platform_libs.v0.data_interfaces +``` + +Your charm directory should now contain the structure below: + +```text +lib +└── charms + └── data_platform_libs + └── v0 + └── data_interfaces.py +``` + +Well done, you've got everything you need to set up a database relation! + + +## Define the charm relation interface + + + +Now, time to define the charm relation interface. + +First, find out the name of the interface that PostgreSQL offers for other charms to connect to it. According to the [documentation of the PostgreSQL charm](https://charmhub.io/postgresql-k8s?channel=14/stable), the interface is called `postgresql_client`. + +Next, open the `charmcraft.yaml` file of your charm and, before the `containers` section, define a relation endpoint using a `requires` block, as below. This endpoint says that our charm is requesting a relation called `database` over an interface called `postgresql_client` with a maximum number of supported connections of 1. (Note: Here, `database` is a custom relation name, though in general we recommend sticking to default recommended names for each charm.) + +```yaml +requires: + database: + interface: postgresql_client + limit: 1 +``` + +That will tell `juju` that our charm can be integrated with charms that provide the same `postgresql_client` interface, for example, the official PostgreSQL charm. + + +Import the database interface libraries and define database event handlers + +We now need to implement the logic that wires our application to a database. When a relation between our application and the data platform is formed, the provider side (i.e., the data platform) will create a database for us and it will provide us with all the information we need to connect to it over the relation -- e.g., username, password, host, port, etc. On our side, we nevertheless still need to set the relevant environment variables to point to the database and restart the service. + +To do so, we need to update our charm “src/charm.py” to do all of the following: + +* Import the `DataRequires` class from the interface library; this class represents the relation data exchanged in the client-server communication. +* Define the event handlers that will be called during the relation lifecycle. +* Bind the event handlers to the observed relation events. + +### Import the database interface libraries + + +First, at the top of the file, import the database interfaces library: + +```python +# Import the 'data_interfaces' library. +# The import statement omits the top-level 'lib' directory +# because 'charmcraft pack' copies its contents to the project root. +from charms.data_platform_libs.v0.data_interfaces import DatabaseCreatedEvent +from charms.data_platform_libs.v0.data_interfaces import DatabaseRequires +``` + +````{important} + +You might have noticed that despite the charm library being placed in the `lib/charms/...`, we are importing it via: + +```python +from charms.data_platform_libs ... +``` +and not + +```python +from lib.charms.data_platform_libs... +``` + +The former is not resolvable by default but everything works fine when the charm is deployed. Why? Because the `dispatch` script in the packed charm sets the `PYTHONPATH` environment variable to include the `lib` directory when it executes your `src/charm.py` code. This tells python it can check the `lib` directory when looking for modules and packages at import time. + + + +If you're experiencing issues with your IDE or just trying to run the `charm.py` file on your own, make sure to set/update `PYTHONPATH` to include `lib` directory as well. + +```bash +# from the charm project directory (~/fastapi-demo), set +export PYTHONPATH=lib +# or update +export PYTHONPATH=lib:$PYTHONPATH +``` + +```` + + +### Add relation event observers + +Next, in the `__init__` method, define a new instance of the 'DatabaseRequires' class. This is required to set the right permissions scope for the PostgreSQL charm. It will create a new user with a password and a database with the required name (below, `names_db`), and limit the user permissions to only this particular database (that is, below, `names_db`). + + +```python +# The 'relation_name' comes from the 'charmcraft.yaml file'. +# The 'database_name' is the name of the database that our application requires. +# See the application documentation in the GitHub repository. +self.database = DatabaseRequires(self, relation_name="database", database_name="names_db") +``` + +Now, add event observers for all the database events: + +```python +# See https://charmhub.io/data-platform-libs/libraries/data_interfaces +framework.observe(self.database.on.database_created, self._on_database_created) +framework.observe(self.database.on.endpoints_changed, self._on_database_created) +``` + +### Fetch the database authentication data + +Now we need to extract the database authentication data and endpoints information. We can do that by adding a `fetch_postgres_relation_data` method to our charm class. Inside this method, we first retrieve relation data from the PostgreSQL using the `fetch_relation_data` method of the `database` object. We then log the retrieved data for debugging purposes. Next we process any non-empty data to extract endpoint information, the username, and the password and return this process data as a dictionary. Finally, we ensure that, if no data is retrieved, we return an empty dictionary, so that the caller knows that the database is not yet ready. + +```python +def fetch_postgres_relation_data(self) -> Dict[str, str]: + """Fetch postgres relation data. + + This function retrieves relation data from a postgres database using + the `fetch_relation_data` method of the `database` object. The retrieved data is + then logged for debugging purposes, and any non-empty data is processed to extract + endpoint information, username, and password. This processed data is then returned as + a dictionary. If no data is retrieved, the unit is set to waiting status and + the program exits with a zero status code.""" + relations = self.database.fetch_relation_data() + logger.debug('Got following database data: %s', relations) + for data in relations.values(): + if not data: + continue + logger.info('New PSQL database endpoint is %s', data['endpoints']) + host, port = data['endpoints'].split(':') + db_data = { + 'db_host': host, + 'db_port': port, + 'db_username': data['username'], + 'db_password': data['password'], + } + return db_data + return {} +``` + +Since `ops` supports Python 3.8, this tutorial used type annotations compatible with 3.8. If you're following along with this chapter, you'll need to import the following from the `typing` module: +```python +from typing import Dict, Optional +``` +```{important} + +The version of Python that your charm will use is determined in your `charmcraft.yaml`. In this case, we've specified Ubuntu 22.04, which means the charm will actually be running on Python 3.10, so we could have used some more recent Python features, like using the builtin `dict` instead of `Dict`, and the `|` operator for unions, allowing us to write (e.g.) `str | None` instead of `Optional[str]`. This will likely be updated in a future version of this tutorial. + +``` + + +### Share the authentication information with your application + +Our application consumes database authentication information in the form of environment variables. Let's update the Pebble service definition with an `environment` key and let's set this key to a dynamic value -- the class property `self.app_environment`. Your `_pebble_layer` property should look as below: + +```python + @property + def _pebble_layer(self) -> ops.pebble.Layer: + """A Pebble layer for the FastAPI demo services.""" + command = ' '.join( + [ + 'uvicorn', + 'api_demo_server.app:app', + '--host=0.0.0.0', + f"--port={self.config['server-port']}", + ] + ) + pebble_layer: ops.pebble.LayerDict = { + 'summary': 'FastAPI demo service', + 'description': 'pebble config layer for FastAPI demo server', + 'services': { + self.pebble_service_name: { + 'override': 'replace', + 'summary': 'fastapi demo', + 'command': command, + 'startup': 'enabled', + 'environment': self.app_environment, + } + }, + } + return ops.pebble.Layer(pebble_layer) +``` + +Now, let's define this property such that, every time it is called, it dynamically fetches database authentication data and also prepares the output in a form that our application can consume, as below: + +```python +@property +def app_environment(self) -> Dict[str, Optional[str]]: + """This property method creates a dictionary containing environment variables + for the application. It retrieves the database authentication data by calling + the `fetch_postgres_relation_data` method and uses it to populate the dictionary. + If any of the values are not present, it will be set to None. + The method returns this dictionary as output. + """ + db_data = self.fetch_postgres_relation_data() + if not db_data: + return {} + env = { + 'DEMO_SERVER_DB_HOST': db_data.get('db_host', None), + 'DEMO_SERVER_DB_PORT': db_data.get('db_port', None), + 'DEMO_SERVER_DB_USER': db_data.get('db_username', None), + 'DEMO_SERVER_DB_PASSWORD': db_data.get('db_password', None), + } + return env +``` + +Finally, let's define the method that is called on the database created event: + +```python +def _on_database_created(self, event: DatabaseCreatedEvent) -> None: + """Event is fired when postgres database is created.""" + self._update_layer_and_restart() +``` + +The diagram below illustrates the workflow for the case where the database integration exists and for the case where it does not: + +![Integrate your charm with PostgreSQL](integrate_your_charm_with_postgresql.png) + + +## Update the unit status to reflect the integration state + +Now that the charm is getting more complex, there are many more cases where the unit status needs to be set. It's often convenient to do this in a more declarative fashion, which is where the collect-status event can be used. + +> Read more: {ref}`Events > Collect App Status and Collect Unit Status ` + +In your charm's `__init__` add a new observer: + +```python +framework.observe(self.on.collect_unit_status, self._on_collect_status) +``` + +And define a method that does the various checks, adding appropriate statuses. The library will take care of selecting the 'most significant' status for you. + +```python +def _on_collect_status(self, event: ops.CollectStatusEvent) -> None: + port = self.config['server-port'] + if port == 22: + event.add_status(ops.BlockedStatus('Invalid port number, 22 is reserved for SSH')) + if not self.model.get_relation('database'): + # We need the user to do 'juju integrate'. + event.add_status(ops.BlockedStatus('Waiting for database relation')) + elif not self.database.fetch_relation_data(): + # We need the charms to finish integrating. + event.add_status(ops.WaitingStatus('Waiting for database relation')) + try: + status = self.container.get_service(self.pebble_service_name) + except (ops.pebble.APIError, ops.ModelError): + event.add_status(ops.MaintenanceStatus('Waiting for Pebble in workload container')) + else: + if not status.is_running(): + event.add_status(ops.MaintenanceStatus('Waiting for the service to start up')) + # If nothing is wrong, then the status is active. + event.add_status(ops.ActiveStatus()) +``` + +We also want to clean up the code to remove the places where we're setting the status outside of this method, other than anywhere we're wanting a status to show up *during* the event execution (such as `MaintenanceStatus`). In `_on_config_changed`, change the port 22 check to: + +```python + if port == 22: + # The collect-status handler will set the status to blocked. + logger.debug('Invalid port number, 22 is reserved for SSH;) +``` + +And remove the `self.unit.status = WaitingStatus` line from `_update_layer_and_restart` (similarly replacing it with a logging line if you prefer). + +## Validate your charm + + +Time to check the results! + +First, repack and refresh your charm: + +```text +charmcraft pack +juju refresh \ + --path="./demo-api-charm_ubuntu-22.04-amd64.charm" \ + demo-api-charm --force-units --resource \ + demo-server-image=ghcr.io/canonical/api_demo_server:1.0.1 +``` + +Next, deploy the `postgresql-k8s` charm: + +```text +juju deploy postgresql-k8s --channel=14/stable --trust +``` + +Now, integrate our charm with the newly deployed `postgresql-k8s` charm: + +```text +juju integrate postgresql-k8s demo-api-charm +``` + +> Read more: [Integration](https://juju.is/docs/olm/integration), [`juju integrate`](https://juju.is/docs/olm/juju-integrate) + + +Finally, run: + +```text +juju status --relations --watch 1s +``` + +You should see both applications get to the `active` status, and also that the `postgresql-k8s` charm has a relation to the `demo-api-charm` over the `postgresql_client` interface, as below: + +```text +Model Controller Cloud/Region Version SLA Timestamp +charm-model tutorial-controller microk8s/localhost 3.0.0 unsupported 13:50:39+01:00 + +App Version Status Scale Charm Channel Rev Address Exposed Message +demo-api-charm 0.0.9 active 1 demo-api-charm 1 10.152.183.233 no +postgresql-k8s active 1 postgresql-k8s 14/stable 29 10.152.183.195 no Primary + +Unit Workload Agent Address Ports Message +demo-api-charm/0* active idle 10.1.157.90 +postgresql-k8s/0* active idle 10.1.157.92 Primary + +Relation provider Requirer Interface Type Message +postgresql-k8s:database demo-api-charm:database postgresql_client regular +postgresql-k8s:database-peers postgresql-k8s:database-peers postgresql_peers peer +postgresql-k8s:restart postgresql-k8s:restart rolling_op peer +``` + +The relation appears to be up and running, but we should also test that it's working as intended. First, let's try to write something to the database by posting some name to the database via API using `curl` as below -- where `10.1.157.90` is a pod IP and `8000` is our app port. You can repeat the command for multiple names. + +```text +curl -X 'POST' \ + 'http://10.1.157.90:8000/addname/' \ + -H 'accept: application/json' \ + -H 'Content-Type: application/x-www-form-urlencoded' \ + -d 'name=maksim' +``` + +```{important} + +If you changed the `server-port` config value in the previous section, don't forget to change it back to 8000 before doing this! + +``` + +Second, let's try to read something from the database by running: + +```text +curl 10.1.157.90:8000/names +``` + +This should produce something similar to the output below (of course, with the names that *you* decided to use): + +```text +{"names":{"1":"maksim","2":"simon"}} +``` + +Congratulations, your integration with PostgreSQL is functional! + +## Review the final code + +For the full code see: [04_integrate_with_psql](https://github.com/canonical/juju-sdk-tutorial-k8s/tree/04_integrate_with_psql) + +For a comparative view of the code before and after this doc see: [Comparison](https://github.com/canonical/juju-sdk-tutorial-k8s/compare/03_set_workload_version...04_integrate_with_psql) + +> **See next: {ref}`Preserve your charm's data `** diff --git a/docs/tutorial/from-zero-to-hero-write-your-first-kubernetes-charm/make-your-charm-configurable.md b/docs/tutorial/from-zero-to-hero-write-your-first-kubernetes-charm/make-your-charm-configurable.md new file mode 100644 index 000000000..87ceadb39 --- /dev/null +++ b/docs/tutorial/from-zero-to-hero-write-your-first-kubernetes-charm/make-your-charm-configurable.md @@ -0,0 +1,211 @@ +(make-your-charm-configurable)= +# Make your charm configurable + +> {ref}`From Zero to Hero: Write your first Kubernetes charm ` > Make your charm configurable +> +> **See previous: {ref}`Create a minimal Kubernetes charm `** + + +````{important} + +This document is part of a series, and we recommend you follow it in sequence. However, you can also jump straight in by checking out the code from the previous branches: + +```bash +git clone https://github.com/canonical/juju-sdk-tutorial-k8s.git +cd juju-sdk-tutorial-k8s +git checkout 01_create_minimal_charm +git checkout -b 02_make_your_charm_configurable +``` + +```` + +A charm might have a specific configuration that the charm developer might want to expose to the charm user so that the latter can change specific settings during runtime. + +As a charm developer, it is thus important to know how to make your charm configurable. + +This can be done by defining a charm configuration in a file called `charmcraft.yaml` and then adding configuration event handlers ('hooks') in the `src/charm.py` file. + +In this part of the tutorial you will update your charm to make it possible for a charm user to change the port on which the workload application is available. + + + +## Define the configuration options + + +To begin with, let's define the options that will be available for configuration. + +In the `charmcraft.yaml` file you created earlier, define a configuration option, as below. The name of your configurable option is going to be `server-port`. The `default` value is `8000` -- this is the value you're trying to allow a charm user to configure. + + + +```yaml +config: + options: + server-port: + default: 8000 + description: Default port on which FastAPI is available + type: int +``` + +## Define the configuration event handlers + + + +Open your `src/charm.py` file. + +In the `__init__` function, add an observer for the `config_changed` event and pair it with an `_on_config_changed` handler: + +```python +framework.observe(self.on.config_changed, self._on_config_changed) +``` + +Now, define the handler, as below. First, read the `self.config` attribute to get the new value of the setting. Then, validate that this value is allowed (or block the charm otherwise). Next, let's log the value to the logger. Finally, since configuring something like a port affects the way we call our workload application, we also need to update our pebble configuration, which we will do via a newly created method `_update_layer_and_restart` that we will define shortly. + +```python +def _on_config_changed(self, event: ops.ConfigChangedEvent) -> None: + port = self.config['server-port'] # see charmcraft.yaml + + if port == 22: + self.unit.status = ops.BlockedStatus('invalid port number, 22 is reserved for SSH') + return + + logger.debug("New application port is requested: %s", port) + self._update_layer_and_restart() +``` + +```{caution} + +A charm does not know which configuration option has been changed. Thus, make sure to validate all the values. This is especially important since multiple values can be changed in one call. + +``` + +In the `__init__` function, add a new attribute to define a container object for your workload: + +```python +# see 'containers' in charmcraft.yaml +self.container = self.unit.get_container('demo-server') +``` + +Create a new method, as below. This method will get the current Pebble layer configuration and compare the new and the existing service definitions -- if they differ, it will update the layer and restart the service. + +```python +def _update_layer_and_restart(self) -> None: + """Define and start a workload using the Pebble API. + + You'll need to specify the right entrypoint and environment + configuration for your specific workload. Tip: you can see the + standard entrypoint of an existing container using docker inspect + Learn more about interacting with Pebble at https://juju.is/docs/sdk/pebble + Learn more about Pebble layers at + https://canonical-pebble.readthedocs-hosted.com/en/latest/reference/layers + """ + + # Learn more about statuses in the SDK docs: + # https://juju.is/docs/sdk/status + self.unit.status = ops.MaintenanceStatus('Assembling Pebble layers') + try: + # Get the current pebble layer config + services = self.container.get_plan().to_dict().get('services', {}) + if services != self._pebble_layer.to_dict().get('services', {}): + # Changes were made, add the new layer + self.container.add_layer('fastapi_demo', self._pebble_layer, combine=True) + logger.info("Added updated layer 'fastapi_demo' to Pebble plan") + + self.container.restart(self.pebble_service_name) + logger.info(f"Restarted '{self.pebble_service_name}' service") + + self.unit.status = ops.ActiveStatus() + except ops.pebble.APIError: + self.unit.status = ops.MaintenanceStatus('Waiting for Pebble in workload container') +``` + +Now, crucially, update the `_pebble_layer` property to make the layer definition dynamic, as shown below. This will replace the static port `8000` with `f"--port={self.config['server-port']}"`. + +```python +command = ' '.join( + [ + 'uvicorn', + 'api_demo_server.app:app', + '--host=0.0.0.0', + f"--port={self.config['server-port']}", + ] +) +``` + +As you may have noticed, the new `_update_layer_and_restart` method looks like a more advanced variant of the existing `_on_demo_server_pebble_ready` method. Remove the body of the `_on_demo_server_pebble_ready` method and replace it a call to `_update_layer_and_restart` like this: + +```python +def _on_demo_server_pebble_ready(self, event: ops.PebbleReadyEvent) -> None: + self._update_layer_and_restart() +``` + +## Validate your charm + +First, repack and refresh your charm: + +``` +charmcraft pack +juju refresh \ + --path="./demo-api-charm_ubuntu-22.04-amd64.charm" \ + demo-api-charm --force-units --resource \ + demo-server-image=ghcr.io/canonical/api_demo_server:1.0.1 +``` + + +Now, check the available configuration options: + +```text +juju config demo-api-charm +``` + +Our newly defined `server-port` option is there. Let's try to configure it to something else, e.g., `5000`: + +```text +juju config demo-api-charm server-port=5000 +``` + +Now, let's validate that the app is actually running and reachable on the new port by sending the HTTP request below, where `10.1.157.74` is the IP of our pod and `5000` is the new application port: + +```text +curl 10.1.157.74:5000/version +``` + +You should see JSON string with the version of the application: `{"version":"1.0.0"}` + +Let's also verify that our invalid port number check works by setting the port to `22` and then running `juju status`: + +```text +juju config demo-api-charm server-port=22 +juju status +``` + +As expected, the application is indeed in the `blocked` state: + +```text +Model Controller Cloud/Region Version SLA Timestamp +charm-model tutorial-controller microk8s/localhost 3.0.0 unsupported 18:19:24+01:00 + +App Version Status Scale Charm Channel Rev Address Exposed Message +demo-api-charm blocked 1 demo-api-charm 2 10.152.183.215 no invalid port number, 22 is reserved for SSH + +Unit Workload Agent Address Ports Message +demo-api-charm/0* blocked idle 10.1.157.74 invalid port number, 22 is reserved for SSH +``` + + +Congratulations, you now know how to make your charm configurable! + + +## Review the final code + +For the full code see: [02_make_your_charm_configurable](https://github.com/canonical/juju-sdk-tutorial-k8s/tree/02_make_your_charm_configurable) + +For a comparative view of the code before and after this doc see: [Comparison](https://github.com/canonical/juju-sdk-tutorial-k8s/compare/01_create_minimal_charm...02_make_your_charm_configurable) + + +> **See next: {ref}`Expose the version of the application behind your charm `** diff --git a/docs/tutorial/from-zero-to-hero-write-your-first-kubernetes-charm/observe-your-charm-with-cos-lite.md b/docs/tutorial/from-zero-to-hero-write-your-first-kubernetes-charm/observe-your-charm-with-cos-lite.md new file mode 100644 index 000000000..b0b8bb9f3 --- /dev/null +++ b/docs/tutorial/from-zero-to-hero-write-your-first-kubernetes-charm/observe-your-charm-with-cos-lite.md @@ -0,0 +1,470 @@ +(observe-your-charm-with-cos-lite)= +# Observe your charm with COS Lite + +> {ref}`From Zero to Hero: Write your first Kubernetes charm ` > Observe your charm with COS Lite +> +> **See previous: {ref}`Expose your charm's operational tasks via actions `** + + +````{important} + +This document is part of a series, and we recommend you follow it in sequence. However, you can also jump straight in by checking out the code from the previous branches: + + +```text +git clone https://github.com/canonical/juju-sdk-tutorial-k8s.git +cd juju-sdk-tutorial-k8s +git checkout 06_create_actions +git checkout -b 07_cos_integration +``` + +```` + +In a production deployment it is essential to observe and monitor the health of your application. A charm user will want to be able to collect real time metrics and application logs, set up alert rules, and visualise any collected data in a neat form on a dashboard. + +Our application is prepared for that -- as you might recall, it uses [`starlette-exporter`](https://pypi.org/project/starlette-exporter/) to generate real-time application metrics and to expose them via a `/metrics` endpoint that is designed to be scraped by [Prometheus](https://prometheus.io/). As a charm developer, you'll want to use that to make your charm observable. + +In the charming universe, what you would do is deploy the existing [Canonical Observability Stack (COS) lite bundle](https://charmhub.io/cos-lite) -- a convenient collection of charms that includes all of [Prometheus](https://charmhub.io/prometheus-k8s), [Loki](https://charmhub.io/loki-k8s), and [Grafana](https://charmhub.io/grafana-k8s) -- and then integrate your charm with Prometheus to collect real-time application metrics; with Loki to collect application logs; and with Grafana to create dashboards and visualise collected data. + + +In this part of the tutorial we will follow this process to collect various metrics and logs about your application and visualise them on a dashboard. + +## Integrate with Prometheus + +Follow the steps below to make your charm capable of integrating with the existing [Prometheus](https://charmhub.io/prometheus-k8s) charm. This will enable your charm user to collect real-time metrics about your application. + + +### Fetch the Prometheus interface libraries + +Ensure you're in your Multipass Ubuntu VM, in your charm project directory. + +Then, satisfy the interface library requirement of the Prometheus charm by fetching the [`prometheus_scrape`](https://charmhub.io/prometheus-k8s/libraries/prometheus_scrape) library: + +```text +ubuntu@charm-dev:~/fastapi-demo$ charmcraft fetch-lib charms.prometheus_k8s.v0.prometheus_scrape +``` + +Also satisfy the dependency requirements of the `prometheus_scrape` library by fetching the [`juju_topology`](https://charmhub.io/observability-libs/libraries/juju_topology) library: + +```text +ubuntu@charm-dev:~/fastapi-demo$ charmcraft fetch-lib charms.observability_libs.v0.juju_topology +``` + +Your charm directory should now include the structure below: + +```text +lib +└── charms + ├── observability_libs + │ └── v0 + │ └── juju_topology.py + └── prometheus_k8s + └── v0 + └── prometheus_scrape.py +``` + +Note: When you rebuild your charm with `charmcraft pack`, Charmcraft will copy the contents of the top `lib` directory to the project root. Thus, to import this library in your code, use just `charms.prometheus_k8s.v0.prometheus_scrape`. + +### Define the Prometheus relation interface + +In your `charmcraft.yaml` file, before the `peers` block, add a `provides` endpoint with relation name `metrics-endpoint` and interface name `prometheus_scrape`, as below. This declares that your charm can offer services to other charms over the `prometheus-scrape` interface. In short, that your charm is open to integrations with, for example, the official Prometheus charm. (Note: `metrics-endpoint` is the default relation name recommended by the `prometheus_scrape` interface library.) + +```yaml +provides: + metrics-endpoint: + interface: prometheus_scrape +``` + +## Import the Prometheus interface libraries and set up Prometheus scraping + +In your `src/charm.py` file, do the following: + +First, at the top of the file, import the `prometheus_scrape` library: + +```text +from charms.prometheus_k8s.v0.prometheus_scrape import MetricsEndpointProvider +``` + +Now, in your charm's `__init__` method, initialise the `MetricsEndpointProvider` instance with the desired scrape target, as below. Note that this uses the relation name that you specified earlier in the `charmcraft.yaml` file. Also, reflecting the fact that you've made your charm's port configurable (see previous chapter {ref}`Make the charm configurable `), the target job is set to be consumed from config. The URL path is not included because it is predictable (defaults to /metrics), so the Prometheus library uses it automatically. The last line, which sets the `refresh_event` to the `config_change` event, ensures that the Prometheus charm will change its scraping target every time someone changes the port configuration. Overall, this code will allow your application to be scraped by Prometheus once they've been integrated. + +```python +self._prometheus_scraping = MetricsEndpointProvider( + self, + relation_name="metrics-endpoint", + jobs={ref}`{"static_configs": [{"targets": [f"*:{self.config['server-port']}"]}]}], + refresh_event=self.on.config_changed, +) +``` + + + +Congratulations, your charm is ready to be integrated with Prometheus! + +## Integrate with Loki + +Follow the steps below to make your charm capable of integrating with the existing [Loki](https://charmhub.io/loki-k8s) charm. This will enable your charm user to collect application logs. + +### Fetch the Loki interface libraries + + + +Ensure you're in your Multipass Ubuntu VM, in your charm folder. + +Then, satisfy the interface library requirements of the Loki charm by fetching the {ref}``loki_push_api` <7814md>` library: + +```text +ubuntu@charm-dev:~/fastapi-demo$ charmcraft fetch-lib charms.loki_k8s.v0.loki_push_api +``` + +This should add to your charm directory the structure below: + +```text +lib +└── charms + ├── loki_k8s + │ └── v0 + │ └── loki_push_api.py +``` + +Note: The `loki_push_api` library also depends on the [`juju_topology`](https://charmhub.io/observability-libs/libraries/juju_topology) library, but you have already fetched it above for Prometheus. + +Note: When you rebuild your charm with `charmcraft pack`, Charmcraft will copy the contents of the top `lib` directory to the project root. Thus, to import this library in your code, use just `charms.loki_k8s.v0.loki_push_api`. + +### Define the Loki relation interface + +In your `charmcraft.yaml` file, beneath your existing `requires` endpoint, add another `requires` endpoint with relation name `log-proxy` and interface name `loki_push_api`. This declares that your charm can optionally make use of services from other charms over the `loki_push_api` interface. In short, that your charm is open to integrations with, for example, the official Loki charm. (Note: `log-proxy` is the default relation name recommended by the `loki_push_api` interface library.) + + +```yaml +requires: + database: + interface: postgresql_client + limit: 1 + log-proxy: + interface: loki_push_api + limit: 1 +``` + + +## Import the Loki interface libraries and set up the Loki API + +In your `src/charm.py` file, do the following: + +First, import the `loki_push_api` lib: + +```python +from charms.loki_k8s.v0.loki_push_api import LogProxyConsumer +``` + +Then, in your charm's `__init__` method, initialise the `LogProxyConsumer` instance with the defined log files, as shown below. The `log-proxy` relation name comes from the `charmcraft.yaml` file and the`demo_server.log` file is the file where the application dumps logs. Overall this code ensures that your application can push logs to Loki (or any other charms that implement the `loki_push_api`). + +```python +self._logging = LogProxyConsumer( + self, relation_name="log-proxy", log_files=["demo_server.log"] +) +``` + +Congratulations, your charm can now also integrate with Loki! + +## Integrate with Grafana + +Follow the steps below to make your charm capable of integrating with the existing [Grafana](https://charmhub.io/grafana-k8s) charm. This will allow your charm user to visualise the data collected from Prometheus and Loki. + +### Fetch the Grafana interface libraries + +Ensure you're in your Multipass Ubuntu VM, in your charm folder. + +Then, satisfy the interface requirement of the Grafana charm by fetching the [grafana_dashboard](https://charmhub.io/grafana-k8s/libraries/grafana_dashboard) library: + +```text +ubuntu@charm-dev:~/fastapi-demo$ charmcraft fetch-lib charms.grafana_k8s.v0.grafana_dashboard +``` + +Your charm directory should now include the structure below: + +```text +lib +└── charms + ├── grafana_k8s + │ └── v0 + │ └── grafana_dashboard.py +``` + +Note: When you rebuild your charm with `charmcraft pack`, Charmcraft will copy the contents of the top `lib` directory to the project root. Thus, to import this library in your code, use just `charms.grafana_k8s.v0.grafana_dashboard`. + +Note: The `grafana_dashboard` library also depends on the [`juju_topology`](https://charmhub.io/observability-libs/libraries/juju_topology) library, but you have already fetched it above for Prometheus. + +### Define the Grafana relation interface + +In your `charmcraft.yaml` file, add another `provides` endpoint with relation name `grafana-dashboard` and interface name `grafana_dashboard`, as below. This declares that your charm can offer services to other charms over the `grafana-dashboard` interface. In short, that your charm is open to integrations with, for example, the official Grafana charm. (Note: Here `grafana-dashboard` endpoint is the default relation name recommended by the `grafana_dashboard` library.) + +```yaml +provides: + metrics-endpoint: + interface: prometheus_scrape + grafana-dashboard: + interface: grafana_dashboard +``` + +### Import the Grafana interface libraries and set up the Grafana dashboards + +In your `src/charm.py` file, do the following: + +First, at the top of the file, import the `grafana_dashboard` lib: + +```python +from charms.grafana_k8s.v0.grafana_dashboard import GrafanaDashboardProvider +``` + +Now, in your charm's `__init__` method, initialise the `GrafanaDashboardProvider` instance, as below. The `grafana-dashboard` is the relation name you defined earlier in your `charmcraft.yaml` file. Overall, this code states that your application supports the Grafana interface. + +```python +# Provide grafana dashboards over a relation interface +self._grafana_dashboards = GrafanaDashboardProvider(self, relation_name="grafana-dashboard") +``` + +Now, in your `src` directory, create a subdirectory called `grafana_dashboards` and, in this directory, create a file called `FastAPI-Monitoring.json.tmpl` with the following content: +[FastAPI-Monitoring.json.tmpl|attachment](https://discourse.charmhub.io/uploads/short-url/6hSGcAA6n20qyStzLFeekgkzqCc.tmpl) (7.7 KB) . Once your charm has been integrated with Grafana, the `GrafanaDashboardProvider` you defined just before will take this file as well as any other files defined in this directory and put them into a Grafana files tree to be read by Grafana. + +```{important} + +**How to build a Grafana dashboard is beyond the scope of this tutorial. However, if you'd like to get a quick idea:** The dashboard template file was created by manually building a Grafana dashboard using the Grafana web UI, then exporting it to a JSON file and updating the `datasource` `uid` for Prometheus and Loki from constant values to the dynamic variables `"${prometheusds}"` and `"${lokids}"`, respectively. + +``` + +## Specify packages required to build + +When packing a charm, charmcraft uses source distributions for the dependencies. When a charm has a dependency that includes a binary package charmcraft will build the package, but may require additional packages to be installed. + +The `cos-lite` packages include a dependency that has Rust code, and the default charmcraft build environment does not have a Rust compiler, so you need to instruct charmcraft to install one for the build. In your `charmcraft.yaml` file, add a new `parts` section: + +```yaml +parts: + charm: + build-packages: + # Required for the cos-lite packages, which have a Rust dependency. + - cargo +``` + +## Validate your charm + +Open a shell in your Multipass Ubuntu VM, navigate inside your project directory, and run all of the following. + +First, repack and refresh your charm: + +```text +charmcraft pack +juju refresh \ + --path="./demo-api-charm_ubuntu-22.04-amd64.charm" \ + demo-api-charm --force-units --resource \ + demo-server-image=ghcr.io/canonical/api_demo_server:1.0.1 +``` + +Next, test your charm's ability to integrate with Prometheus, Loki, and Grafana by following the steps below. + + +### Deploy COS Lite + +Create a Juju model called `cos-lite` and, to this model, deploy the Canonical Observability Stack bundle [`cos-lite`](https://charmhub.io/topics/canonical-observability-stack), as below. This will deploy all the COS applications (`alertmanager`, `catalogue`, `grafana`, `loki`, `prometheus`, `traefik`), already suitably integrated with one another. Note that these also include the applications that you've been working to make your charm integrate with -- Prometheus, Loki, and Grafana. + +```text +juju add-model cos-lite +juju deploy cos-lite --trust +``` + +```{important} + + +**Why put COS Lite in a separate model?** Because (1) it is always a good idea to separate logically unrelated applications in different models and (2) this way you can observe applications across all your models. PS In a production-grade scenario you would actually even want to put your COS Lite in a separate *cloud* (i.e., Kubernetes cluster). This is recommended, for example, to ensure proper hardware resource allocation. + +``` + + +### Expose the application integration endpoints + +Once all the COS Lite applications are deployed and settled down (you can monitor this by using `juju status --watch 2s`), expose the integration points you are interested in for your charm -- `loki:logging`, `grafana-dashboard`, and `metrics-endpoint` -- as below. + +```text +juju offer prometheus:metrics-endpoint +juju offer loki:logging +juju offer grafana:grafana-dashboard +``` + +Validate that the offers have been successfully created by running: + +```text +juju find-offers cos-lite +``` + +You should something similar to the output below: + +```text +Store URL Access Interfaces +tutorial-controller admin/cos-lite.loki admin loki_push_api:logging +tutorial-controller admin/cos-lite.prometheus admin prometheus_scrape:metrics-endpoint +tutorial-controller admin/cos-lite.grafana admin grafana_dashboard:grafana-dashboard +``` + +As you might notice from your knowledge of Juju, this is essentially preparing these endpoints, which exist in the `cos-lite` model, for a cross-model relation with your charm, which you've deployed to the `charm-model` model. + +## Integrate your charm with COS Lite + +Now switch back to the charm model and integrate your charm with the exposed endpoints, as below. This effectively integrates your application with Prometheus, Loki, and Grafana. + +```text +juju switch charm-model +juju integrate demo-api-charm admin/cos-lite.grafana +juju integrate demo-api-charm admin/cos-lite.loki +juju integrate demo-api-charm admin/cos-lite.prometheus +``` + +

Access your applications from the host machine

 + +```{important} + +The power of Grafana lies in the way it allows you to visualise metrics on a dashboard. Thus, in the general case you will want to open the Grafana Web UI in a web browser. However, you are now working in a headless VM that does not have any user interface. This means that you will need to open Grafana in a web browser on your host machine. To do this, you will need to add IP routes to the Kubernetes (MicroK8s) network inside of our VM. You can skip this step if you have decided to follow this tutorial directly on your host machine. + +``` + + +First, run: + +```text +juju status -m cos-lite +``` + +This should result in an output similar to the one below: + +```text +Model Controller Cloud/Region Version SLA Timestamp +cos-lite tutorial-controller microk8s/localhost 3.0.0 unsupported 18:05:07+01:00 + +App Version Status Scale Charm Channel Rev Address Exposed Message +alertmanager 0.23.0 active 1 alertmanager-k8s stable 36 10.152.183.70 no +catalogue active 1 catalogue-k8s stable 4 10.152.183.19 no +grafana 9.2.1 active 1 grafana-k8s stable 52 10.152.183.132 no +loki 2.4.1 active 1 loki-k8s stable 47 10.152.183.207 no +prometheus 2.33.5 active 1 prometheus-k8s stable 79 10.152.183.196 no +traefik active 1 traefik-k8s stable 93 10.152.183.83 no +``` + +From this output, from the `Address` column, retrieve the IP address for each app to obtain the Kubernetes service IP address range. Make a note of each as well as the range. (In our output we got the `10.152.183.0-10.152.183.255` range.) + +```{caution} + +Do not mix up Apps and Units -- Units represent Kubernetes pods while Apps represent Kubernetes Services. Note: The charm should be programmed to support Services. + +``` + +Now open a terminal on your host machine and run: + +```text +multipass info charm-dev +``` + +This should result in an output similar to the one below: + +```text +Name: charm-dev +State: Running +IPv4: 10.112.13.157 + 10.49.132.1 + 10.1.157.64 +Release: Ubuntu 22.04.1 LTS +Image hash: 1d24e397489d (Ubuntu 22.04 LTS) +Load: 0.31 0.25 0.28 +Disk usage: 15.9G out of 19.2G +Memory usage: 2.1G out of 7.8G +Mounts: /home/maksim/fastapi-demo => ~/fastapi-demo + UID map: 1000:default + GID map: 1000:default +``` + +From this output, retrieve your Multipass Ubuntu VM's network IP address. In our case it is `10.112.13.157`. + +Now, also on your host machine, run the code below. Until the next reboot, this will forward all the traffic for your Kubernetes Service IP range via the network on your VM. This will allow you, for example, to view your Grafana dashboards in a web browser inside your VM, as we do in the next step. + +```text +sudo ip route add 10.152.183.0/24 via 10.112.13.157 +``` + +## Log in to Grafana + +In a terminal inside your VM, do all of the following: + +First, run `juju status` again to retrieve the IP address of your Grafana service. For us it is `http://10.152.183.132:3000` (see the output above). + +Now, use `juju run` to retrieve your Grafana password, as shown below. + +```text +juju run grafana/0 -m cos-lite get-admin-password --wait 1m +``` + +Now, on your host machine, open a web browser, enter the Grafana IP address, and use the username "admin" and your Grafana password to log in. + +### Inspect the dashboards + +In your Grafana web page, do all of the following: + +Click `FastAPI Monitoring`. You should see the Grafana Dashboard that we uploaded to the `grafana_dashboards` directory of your charm. + + + +Next, in the `Juju model` drop down field, select `charm-model`. + +Now, call a couple of API points on the application, as below. To produce some successful requests and some requests with code 500 (internal server error), call several times, in any order. + +```text +curl 10.1.157.94:8000/names +``` + +and + +```text +curl 10.1.157.94:8000/error +``` + +where `10.1.157.94` is the IP of our application unit (pod). + + + +In a while you should see the following data appearing on the dashboard: + +1. HTTP Request Duration Percentiles. This dashboard is based on the data from Prometheus and will allow you to see what fraction of requests takes what amount of time. +2. Percent of failed requests per 2 minutes time frame. In your case this will be a ratio of all the requests and the requests submitted to the `/error` path (i.e., the ones that cause the Internal Server Error). +3. Logs from your application that were collected by Loki and forwarded to Grafana. Here you can see some INFO level logs and ERROR logs with traceback from Python when you were calling the `/error` path. + + +![Observe your charm with COS Lite](observe_your_charm_with_cos_lite.png) + +```{important} + +If you are interested in the Prometheus metrics produced by your application that were used to build these dashboards you can run following command in your VM: `curl :8000/metrics` +Also, you can reach Prometheus in your web browser (similar to Grafana) at `http://:9090/graph` . + +``` + + +## Review the final code + + +For the full code see: [07_cos_integration](https://github.com/canonical/juju-sdk-tutorial-k8s/tree/07_cos_integration) + +For a comparative view of the code before and after this doc see: [Comparison](https://github.com/canonical/juju-sdk-tutorial-k8s/compare/06_create_actions...07_cos_integration) + +> **See next: {ref}`Write units tests for your charm `** + diff --git a/docs/tutorial/from-zero-to-hero-write-your-first-kubernetes-charm/open-a-kubernetes-port-in-your-charm.md b/docs/tutorial/from-zero-to-hero-write-your-first-kubernetes-charm/open-a-kubernetes-port-in-your-charm.md new file mode 100644 index 000000000..a0588e693 --- /dev/null +++ b/docs/tutorial/from-zero-to-hero-write-your-first-kubernetes-charm/open-a-kubernetes-port-in-your-charm.md @@ -0,0 +1,359 @@ +(open-a-kubernetes-port-in-your-charm)= +# Open a Kubernetes port in your charm + +> {ref}`From Zero to Hero: Write your first Kubernetes charm ` > Open a Kubernetes port in your charm +> +> **See previous: {ref}`Write integration tests for your charm `** + +````{important} + +This document is part of a series, and we recommend you follow it in sequence. However, you can also jump straight in by checking out the code from the previous branches: + +```bash +git clone https://github.com/canonical/juju-sdk-tutorial-k8s.git +cd juju-sdk-tutorial-k8s +git checkout 10_integration_testing +git checkout -b 11_open_port_k8s_service +``` + +```` + +A deployed charm should be consistently accessible via a stable URL on a cloud. + +However, our charm is currently accessible only at the IP pod address and, if the pod gets recycled, the IP address will change as well. + +> See earlier chapter: {ref}`Make your charm configurable ` + +In Kubernetes you can make a service permanently reachable under a stable URL on the cluster by exposing a service port via the `ClusterIP`. In Juju 3.1+, you can take advantage of this by using the `Unit.set_ports()` method. + +> Read more: [ClusterIP](https://kubernetes.io/docs/concepts/services-networking/service/#type-clusterip) + +In this chapter of the tutorial you will extend the existing `server-port` configuration option to use Juju `open-port` functionality to expose a Kubernetes service port. Building on your experience from the previous testing chapters, you will also write tests to check that the new feature you've added works as intended. + + +## Add a Kubernetes service port to your charm + +In your `src/charm.py` file, do all of the following: + +In the `_on_config_changed` method, add a new method: + +```python +self._handle_ports() +``` + +Then, in the definition of the `FastAPIDemoCharm` class, define the method: + +```python +def _handle_ports(self) -> None: + port = cast(int, self.config['server-port']) + self.unit.set_ports(port) +``` + +> See more: [`ops.Unit.set_ports`](https://ops.readthedocs.io/en/latest/#ops.Unit.set_ports) + + +## Test the new feature + + +### Write a unit test + + +```{important} + +**If you've skipped straight to this chapter:**
Note that it builds on the earlier unit testing chapter. To catch up, see: {ref}`Write unit tests for your charm `. + +``` + +Let's write a unit test to verify that the port is opened. Open `tests/unit/test_charm.py` and add the following test function to the file. + +```python +@pytest.mark.parametrize( + 'port,expected_status', + [ + (22, ops.BlockedStatus('Invalid port number, 22 is reserved for SSH')), + (1234, ops.BlockedStatus('Waiting for database relation')), + ], +) +def test_port_configuration( + monkeypatch, harness: ops.testing.Harness[FastAPIDemoCharm], port, expected_status +): + # Given + monkeypatch.setattr(FastAPIDemoCharm, 'version', '1.0.1') + harness.container_pebble_ready('demo-server') + # When + harness.update_config({'server-port': port}) + harness.evaluate_status() + currently_opened_ports = harness.model.unit.opened_ports() + port_numbers = {port.port for port in currently_opened_ports} + server_port_config = harness.model.config.get('server-port') + unit_status = harness.model.unit.status + # Then + if port == 22: + assert server_port_config not in port_numbers + else: + assert server_port_config in port_numbers + assert unit_status == expected_status +``` + +```{important} + +**Tests parametrisation**
Note that we used the `parametrize` decorator to run a single test against multiple sets of arguments. Adding a new test case, like making sure that the error message is informative given a negative or too big port number, would be as simple as extending the list in the decorator call. +See [How to parametrize fixtures and test functions](https://docs.pytest.org/en/8.0.x/how-to/parametrize.html). + +``` + +Time to run the tests! + +In your Multipass Ubuntu VM shell, run the unit test: + +``` +ubuntu@charm-dev:~/fastapi-demo$ tox -re unit +``` + +If successful, you should get an output similar to the one below: + +```bash +$ tox -re unit +unit: remove tox env folder /home/ubuntu/fastapi-demo/.tox/unit +unit: install_deps> python -I -m pip install cosl 'coverage[toml]' pytest -r /home/ubuntu/fastapi-demo/requirements.txt +unit: commands[0]> coverage run --source=/home/ubuntu/fastapi-demo/src -m pytest --tb native -v -s /home/ubuntu/fastapi-demo/tests/unit +========================================= test session starts ========================================= +platform linux -- Python 3.10.13, pytest-8.0.2, pluggy-1.4.0-- /home/ubuntu/fastapi-demo/.tox/unit/bin/python +cachedir: .tox/unit/.pytest_cache +rootdir: /home/ubuntu/fastapi-demo +collected 3 items + +tests/unit/test_charm.py::test_pebble_layer PASSED +tests/unit/test_charm.py::test_port_configuration[22-expected_status0] PASSED +tests/unit/test_charm.py::test_port_configuration[1234-expected_status1] PASSED + +========================================== 3 passed in 0.21s ========================================== +unit: commands[1]> coverage report +Name Stmts Miss Cover +---------------------------------- +src/charm.py 122 43 65% +---------------------------------- +TOTAL 122 43 65% + unit: OK (6.00=setup[5.43]+cmd[0.49,0.09] seconds) + congratulations :) (6.04 seconds) +``` + +### Write a scenario test + +Let's also write a scenario test! Add this test to your `tests/scenario/test_charm.py` file: + +```python +def test_open_port(monkeypatch: MonkeyPatch): + monkeypatch.setattr('charm.LogProxyConsumer', Mock()) + monkeypatch.setattr('charm.MetricsEndpointProvider', Mock()) + monkeypatch.setattr('charm.GrafanaDashboardProvider', Mock()) + + # Use scenario.Context to declare what charm we are testing. + ctx = scenario.Context( + FastAPIDemoCharm, + meta={ + 'name': 'demo-api-charm', + 'containers': {'demo-server': {}}, + 'peers': {'fastapi-peer': {'interface': 'fastapi_demo_peers'}}, + 'requires': { + 'database': { + 'interface': 'postgresql_client', + } + }, + }, + config={ + 'options': { + 'server-port': { + 'default': 8000, + } + } + }, + actions={ + 'get-db-info': {'params': {'show-password': {'default': False, 'type': 'boolean'}}} + }, + ) + state_in = scenario.State( + leader=True, + relations=[ + scenario.Relation( + endpoint='database', + interface='postgresql_client', + remote_app_name='postgresql-k8s', + local_unit_data={}, + remote_app_data={ + 'endpoints': '127.0.0.1:5432', + 'username': 'foo', + 'password': 'bar', + }, + ), + scenario.PeerRelation( + endpoint='fastapi-peer', + peers_data={'unit_stats': {'started_counter': '0'}}, + ), + ], + containers=[ + scenario.Container(name='demo-server', can_connect=True), + ], + ) + state1 = ctx.run('config_changed', state_in) + assert len(state1.opened_ports) == 1 + assert state1.opened_ports[0].port == 8000 + assert state1.opened_ports[0].protocol == 'tcp' +``` + +In your Multipass Ubuntu VM shell, run your scenario test as below: + +```bash +ubuntu@charm-dev:~/fastapi-demo$ tox -re scenario +``` + +If successful, this should yield: + +```bash +scenario: remove tox env folder /home/ubuntu/fastapi-demo/.tox/scenario +scenario: install_deps> python -I -m pip install cosl 'coverage[toml]' ops-scenario pytest -r /home/ubuntu/fastapi-demo/requirements.txt +scenario: commands[0]> coverage run --source=/home/ubuntu/fastapi-demo/src -m pytest --tb native -v -s /home/ubuntu/fastapi-demo/tests/scenario +========================================= test session starts ========================================= +platform linux -- Python 3.10.13, pytest-8.0.2, pluggy-1.4.0 -- /home/ubuntu/fastapi-demo/.tox/scenario/bin/python +cachedir: .tox/scenario/.pytest_cache +rootdir: /home/ubuntu/fastapi-demo +collected 2 items + +tests/scenario/test_charm.py::test_get_db_info_action PASSED +tests/scenario/test_charm.py::test_open_port PASSED + +========================================== 2 passed in 0.31s ========================================== +scenario: commands[1]> coverage report +Name Stmts Miss Cover +---------------------------------- +src/charm.py 122 22 82% +---------------------------------- +TOTAL 122 22 82% + scenario: OK (6.66=setup[5.98]+cmd[0.59,0.09] seconds) + congratulations :) (6.69 seconds) +``` + +### Write an integration test + +In your `tests/integration` directory, create a `helpers.py` file with the following contents: + +```python +import socket +from pytest_operator.plugin import OpsTest + + +async def get_address(ops_test: OpsTest, app_name: str, unit_num: int = 0) -> str: + """Get the address for a the k8s service for an app.""" + status = await ops_test.model.get_status() + k8s_service_address = status['applications'][app_name].public_address + return k8s_service_address + + +def is_port_open(host: str, port: int) -> bool: + """check if a port is opened in a particular host""" + try: + with socket.create_connection((host, port), timeout=5): + return True # If connection succeeds, the port is open + except (ConnectionRefusedError, TimeoutError): + return False # If connection fails, the port is closed +``` + +In your existing `tests/integration/test_charm.py` file, import the methods defined in `helpers.py`: + +```python +from helpers import is_port_open, get_address +``` + +Now add the test case that will cover open ports: + +```python +@pytest.mark.abort_on_fail +async def test_open_ports(ops_test: OpsTest): + """Verify that setting the server-port in charm's config correctly adjust k8s service + + Assert blocked status in case of port 22 and active status for others + """ + app = ops_test.model.applications.get('demo-api-charm') + + # Get the k8s service address of the app + address = await get_address(ops_test=ops_test, app_name=APP_NAME) + # Validate that initial port is opened + assert is_port_open(address, 8000) + + # Set Port to 22 and validate app going to blocked status with port not opened + await app.set_config({'server-port': '22'}) + (await ops_test.model.wait_for_idle(apps=[APP_NAME], status='blocked', timeout=120),) + assert not is_port_open(address, 22) + + # Set Port to 6789 "Dummy port" and validate app going to active status with port opened + await app.set_config({'server-port': '6789'}) + (await ops_test.model.wait_for_idle(apps=[APP_NAME], status='active', timeout=120),) + assert is_port_open(address, 6789) +``` +In your Multipass Ubuntu VM shell, run the test as below: + +```bash +ubuntu@charm-dev:~/fastapi-demo$ tox -re integration +``` + +This test will take longer as a new model needs to be created. If successful, it should yield something similar to the output below: + +```bash +==================================== 3 passed in 234.15s (0:03:54) ==================================== + integration: OK (254.77=setup[19.55]+cmd[235.22] seconds) + congratulations :) (254.80 seconds) +``` + +## Validate your charm + +Congratulations, you've added a new feature to your charm, and also written tests to ensure that it will work properly. Time to give this feature a test drive! + +In your Multipass VM, repack and refresh your charm as below: + +```bash +ubuntu@charm-dev:~/fastapi-demo$ charmcraft pack +juju refresh \ + --path="./demo-api-charm_ubuntu-22.04-amd64.charm" \ + demo-api-charm --force-units --resource \ + demo-server-image=ghcr.io/canonical/api_demo_server:1.0.1 +``` + +Watch your charm deployment status change until deployment settles down: + +``` +juju status --watch 1s +``` + +Use `kubectl` to list the available services and verify that `demo-api-charm` service exposes the `ClusterIP` on the expected port: + + +```bash +$ kubectl get services -n charm-model +NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE +modeloperator ClusterIP 10.152.183.231 17071/TCP 34m +demo-api-charm-endpoints ClusterIP None 19m +demo-api-charm ClusterIP 10.152.183.92 65535/TCP,8000/TCP 19m +postgresql-k8s-endpoints ClusterIP None 18m +postgresql-k8s ClusterIP 10.152.183.162 5432/TCP,8008/TCP 18m +postgresql-k8s-primary ClusterIP 10.152.183.109 8008/TCP,5432/TCP 18m +postgresql-k8s-replicas ClusterIP 10.152.183.29 8008/TCP,5432/TCP 18m +patroni-postgresql-k8s-config ClusterIP None 17m +``` + +Finally, `curl` the `ClusterIP` to verify that the `version` endpoint responds on the expected port: + +```bash +$ curl 10.152.183.92:8000/version +{"version":"1.0.1"} +``` + +Congratulations, your service now exposes an external port that is independent of any pod / node restarts! + +## Review the final code + +For the full code see: [11_open_port_k8s_service](https://github.com/canonical/juju-sdk-tutorial-k8s/tree/11_open_port_k8s_service) + +For a comparative view of the code before and after this doc see: [Comparison](https://github.com/canonical/juju-sdk-tutorial-k8s/compare/10_integration_testing...11_open_port_k8s_service) + +> **See next: {ref}`Publish your charm on Charmhub `** + diff --git a/docs/tutorial/from-zero-to-hero-write-your-first-kubernetes-charm/preserve-your-charms-data.md b/docs/tutorial/from-zero-to-hero-write-your-first-kubernetes-charm/preserve-your-charms-data.md new file mode 100644 index 000000000..138f2a90a --- /dev/null +++ b/docs/tutorial/from-zero-to-hero-write-your-first-kubernetes-charm/preserve-your-charms-data.md @@ -0,0 +1,170 @@ +(preserve-your-charms-data)= +# Preserve your charm's data + +> {ref}`From Zero to Hero: Write your first Kubernetes charm ` > Preserve your charm's data +> +> **See previous: {ref}`Integrate your charm with PostgreSQL `** + + +````{important} + +This document is part of a series, and we recommend you follow it in sequence. However, you can also jump straight in by checking out the code from the previous branches: + +```bash +git clone https://github.com/canonical/juju-sdk-tutorial-k8s.git +cd juju-sdk-tutorial-k8s +git checkout 04_integrate_with_psql +git checkout -b 05_preserve_charm_data +``` + +```` + + +Charms are stateless applications. That is, they are reinitialised for every event and do not retain information from previous executions. This means that, if an accident occurs and the Kubernetes pod dies, you will also lose any information you may have collected. + +In many cases that is not a problem. However, there are situations where it may be necessary to maintain information from previous runs and to retain the state of the application. As a charm author you should thus know how to preserve state. + +There are a few strategies you can adopt here: + +First, you can use an Ops construct called `Stored State`. With this strategy you can store data on the local unit (at least, so long as your `main` function doesn't set `use_juju_for_storage` to `True`). However, if your Kubernetes pod dies, your unit also dies, and thus also the data. For this reason this strategy is generally not recommended. + +> Read more: {ref}`StoredState <7433md>`, {ref}`StoredState: Uses, Limitations ` + +Second, you can make use of the Juju notion of 'peer relations' and 'data bags' and set up a peer relation data bag. This will help you store the information in the Juju's database backend. + +> Read more: [Peer integrations](https://juju.is/docs/juju/relation#heading--peer) + +Third, when you have confidential data, you can use Juju secrets (from Juju 3.1 onwards). + +> Read more: [Juju | Secret](https://juju.is/docs/juju/secret) + +In this chapter we will adopt the second strategy, that is, we will store charm data in a peer relation databag. (We will explore the third strategy in a different scenario in the next chapter.) We will illustrate this strategy with an artificial example where we save the counter of how many times the application pod has been restarted. + +## Define a peer relation + +The first thing you need to do is define a peer relation. Update the `charmcraft.yaml` file to add a `peers` block before the `requires` block, as below (where `fastapi-peer` is a custom name for the peer relation and `fastapi_demo_peers` is a custom name for the peer relation interface): + +```yaml +peers: + fastapi-peer: + interface: fastapi_demo_peers +``` +> Read more: {ref}`File ‘charmcraft.yaml’ <7433md>` + +## Set and get data from the peer relation databag + +Now, you need a way to set and get data from the peer relation databag. For that you need to update the `src/charm.py` file as follows: + +First, define some helper methods that will allow you to read and write from the peer relation databag: + +```python +@property +def peers(self) -> Optional[ops.Relation]: + """Fetch the peer relation.""" + return self.model.get_relation(PEER_NAME) + +def set_peer_data(self, key: str, data: JSONData) -> None: + """Put information into the peer data bucket instead of `StoredState`.""" + peers = cast(ops.Relation, self.peers) + peers.data[self.app][key] = json.dumps(data) + +def get_peer_data(self, key: str) -> Dict[str, JSONData]: + """Retrieve information from the peer data bucket instead of `StoredState`.""" + if not self.peers: + return {} + data = self.peers.data[self.app].get(key, '') + if not data: + return {} + return json.loads(data) +``` + +This block uses the built-in `json` module of Python, so you need to import that as well. You also need to define a global variable called `PEER_NAME = "fastapi-peer"`, to match the name of the peer relation defined in `charmcraft.yaml` file. We'll also need to import some additional types from `typing`, and define a type alias for JSON data. Update your imports to include the following: + +```python +import json +from typing import Dict, List, Optional, Union, cast +``` +Then define our global and type alias as follows: + +```python +PEER_NAME = 'fastapi-peer' + +JSONData = Union[ + Dict[str, 'JSONData'], + List['JSONData'], + str, + int, + float, + bool, + None, +] +``` + +Next, you need to add a method that updates a counter for the number of times a Kubernetes pod has been started. Let's make it retrieve the current count of pod starts from the 'unit_stats' peer relation data, increment the count, and then update the 'unit_stats' data with the new count, as below: + +```python +def _count(self, event: ops.StartEvent) -> None: + """This function updates a counter for the number of times a K8s pod has been started. + + It retrieves the current count of pod starts from the 'unit_stats' peer relation data, + increments the count, and then updates the 'unit_stats' data with the new count. + """ + unit_stats = self.get_peer_data('unit_stats') + counter = cast(str, unit_stats.get('started_counter', '0')) + self.set_peer_data('unit_stats', {'started_counter': int(counter) + 1}) +``` + +Finally, you need to call this method and update the peer relation data every time the pod is started. For that, define another event observer in the `__init__` method, as below: + +```python +framework.observe(self.on.start, self._count) +``` + +## Validate your charm + +First, repack and refresh your charm: + +```bash +charmcraft pack +juju refresh \ + --path="./demo-api-charm_ubuntu-22.04-amd64.charm" \ + demo-api-charm --force-units --resource \ + demo-server-image=ghcr.io/canonical/api_demo_server:1.0.1 +``` + + +Next, run `juju status` to make sure the application is refreshed and started, then investigate the relation data as below: + +```bash +juju show-unit demo-api-charm/0 +``` + +The output should include the following lines related to our peer relation: + +```bash + relation-info: + - relation-id: 25 + endpoint: fastapi-peer + related-endpoint: fastapi-peer + application-data: + unit_stats: '{"started_counter": 1}' +``` + +Now, simulate a Kubernetes pod crash by deleting the charm pod: + +```bash +microk8s kubectl --namespace=charm-model delete pod demo-api-charm-0 +``` + +Finally, check the peer relation again. You should see that the `started_counter` has been incremented by one. Good job, you've preserved your application data across restarts! + +## Review the final code + + +For the full code see: [05_preserve_charm_data](https://github.com/canonical/juju-sdk-tutorial-k8s/tree/05_preserve_charm_data) + +For a comparative view of the code before and after this doc see: [Comparison](https://github.com/canonical/juju-sdk-tutorial-k8s/compare/04_integrate_with_psql...05_preserve_charm_data) + + +> **See next: {ref}`Expose your charm's operational tasks via actions `** + diff --git a/docs/tutorial/from-zero-to-hero-write-your-first-kubernetes-charm/publish-your-charm-on-charmhub.md b/docs/tutorial/from-zero-to-hero-write-your-first-kubernetes-charm/publish-your-charm-on-charmhub.md new file mode 100644 index 000000000..a08f97613 --- /dev/null +++ b/docs/tutorial/from-zero-to-hero-write-your-first-kubernetes-charm/publish-your-charm-on-charmhub.md @@ -0,0 +1,192 @@ +(publish-your-charm-on-charmhub)= +# Publish your charm on Charmhub + +> {ref}`From Zero to Hero: Write your first Kubernetes charm ` > Pushing your charm to charmhub +> +> **See previous: {ref}`Open a Kubernetes port in your charm `** + +````{important} + +This document is part of a series, and we recommend you follow it in sequence. However, you can also jump straight in by checking out the code from the previous branches: + +```bash +git clone https://github.com/canonical/juju-sdk-tutorial-k8s.git +cd juju-sdk-tutorial-k8s +git checkout 11_open_port_k8s_service +``` + +```` + +In this tutorial you've done a lot of work, and the result is an increasingly functional charm. + +You can enjoy this charm on your own, or pass it around to friends, but why not share it with the whole world? + +The Canonical way to share a charm publicly is to publish it on [Charmhub](https://charmhub.io/). Aside from making your charm more visible, this also means you can deploy it more easily, as Charmhub is the default source for `juju deploy`. Besides, Charmcraft is there to support you every step of the way. + +In this chapter of the tutorial you will use Charmcraft to release your charm on Charmhub. + + +## Log in to Charmhub + +```{caution} + +**You will need an Ubuntu SSO account.**
+If you don't have one yet, sign up on https://login.ubuntu.com/+login + +``` + +```{note} + +Logging into Charmhub is typically a simple matter of running `charmcraft login` . However, here we are within a Multipass VM, so we have to take some extra steps. + +``` + + +On your Multipass VM, run the code below: + +```bash +ubuntu@charm-dev:~/fastapi-demo$ charmcraft login --export ~/secrets.auth +``` + +Once you've put in your login information, you should see something similar to the output below: + +```text +Opening an authorization web page in your browser. +If it does not open, please open this URL: + https://api.jujucharms.com/identity/login?did=48d45d919ca2b897a81470dc5e98b1a3e1e0b521b2fbcd2e8dfd414fd0e3fa96 +``` + +Copy-paste the provided web link into your web browser. Use your Ubuntu SSO to log in. + +When you're done, you should see in your terminal the following: + +```text +Login successful. Credentials exported to '~/secrets.auth'. +``` + +Now set an environment variable with the new token: + +```bash +export CHARMCRAFT_AUTH=$(cat ~/secrets.auth) +``` + +Well done, you're now logged in to Charmhub! + +## Register your charm's name + +On your Multipass VM, generate a random 8-digit hexadecimal hash, then view it in the shell: + +```text +random_hash=$(cat /dev/urandom | tr -dc 'a-f0-9' | head -c 8) +echo "Random 8-digit hash: $random_hash" +``` +```{important} + +Naming your charm is usually less random than that; see {ref}`Charm naming guidelines `. However, here we are in a tutorial setting, so you just need to make sure to pick a unique name, any name. + +``` + +Navigate to the `charmcraft.yaml` file of your charm and update the `name` field with the randomly generated name. + +Once done, prepare the charm for upload by executing `charmcraft pack` . This command will create a compressed file with the updated name prefix, as discussed earlier. + +Now pass this hash as the name to register for your charm on Charmhub: + +```bash +$ charmcraft register +Congrats! You are now the publisher of '' +``` + +You're all set! + +## Upload the charm and its resources + +On your Multipass VM, run the code below. (The argument to `charmcraft upload` is the filepath to the `.charm` file.) + +```text +charmcraft upload _ubuntu-22.04-amd64.charm +Revision 1 of created +``` + +```{note} + +Every time a new binary is uploaded for a charm, a new revision is created on Charmhub. We can verify its current status easily by running `charmcraft revisions `. + +``` + + +Now upload the charm's resource -- in your case, the `demo-server-image` OCI image specified in your charm's `charmcraft.yaml` as follows: + + + +First, pull it locally: + +```text +docker pull ghcr.io/canonical/api_demo_server:1.0.1 +``` + +Then, take note of the image ID: + +```text +docker images ghcr.io/canonical/api_demo_server +``` + +This should output something similar to the output below: + +```text +REPOSITORY TAG IMAGE ID CREATED SIZE +ghcr.io/canonical/api_demo_server 1.0.1 6 months ago 532MB +``` + +Finally, upload the image as below, specifying first the charm name, then the image name, then a flag with the image digest: + +```text +charmcraft upload-resource demo-server-image --image= +``` + +Sample output: + +```text +Revision 1 created of resource 'demo-server-image' for charm ''. +``` + +All set! + +## Release the charm + +Release your charm as below. + +```{important} + +**Do not worry:**
+While releasing a charm to Charmhub gives it a public URL, the charm will not appear in the Charmhub search results until it has passed formal review -- see {ref}`Requirements for public listing <12281md>`. + +``` + + +```text +$ charmcraft release --revision=1 --channel=beta --resource=demo-server-image:1 +Revision 1 of charm '` released to beta +``` + +This releases it into a channel so it can become available for downloading. + +Just in case, also check your charm's status: + +```text +$ charmcraft status +Track Base Channel Version Revision Resources +latest ubuntu 22.04 (amd64) stable - - - + candidate - - - + beta 1 1 demo-server-image (r1) + edge ↑ ↑ ↑ +``` + +Congratulations, your charm has now been published to charmhub.io! + +You can view it at any time at `charmhub.io/`. + + + diff --git a/docs/tutorial/from-zero-to-hero-write-your-first-kubernetes-charm/set-up-your-development-environment.md b/docs/tutorial/from-zero-to-hero-write-your-first-kubernetes-charm/set-up-your-development-environment.md new file mode 100644 index 000000000..752ce81ec --- /dev/null +++ b/docs/tutorial/from-zero-to-hero-write-your-first-kubernetes-charm/set-up-your-development-environment.md @@ -0,0 +1,30 @@ +(set-up-your-development-environment)= +# Set up your development environment + +> {ref}`From zero to hero: Write your first Kubernetes charm ` > Set up your development environment +> +> **See previous: {ref}`Study your application `** + +In this chapter of the tutorial you will set up your development environment. + +You will need a charm directory, the various tools in the charm SDK, Juju, and a Kubernetes cloud. And it’s a good idea if you can do all your work in an isolated development environment. + +You can get all of this by following our generic development setup guide, with some annotations. + +> See [`juju` | Set up your environment automatically](https://juju.is/docs/juju/set-up--tear-down-your-test-environment#tear-down-automatically, with the following changes: +> - At the directory step, call your directory `fastapi-demo`. +> - At the VM setup step, call your VM `charm-dev` and also set up Docker: +> 1. `sudo addgroup --system docker` +> 1. `sudo adduser $USER docker` +> 1. `newgrp docker` +> 1. `sudo snap install docker`. +> - At the cloud selection step, choose `microk8s`. +> - At the mount step: Make sure to read the box with tips on how to edit files locally while running them inside the VM!

+> All set! + + + +Congratulations, your development environment is now ready! + +> **See next: {ref}`Create a minimal Kubernetes charm `** + diff --git a/docs/tutorial/from-zero-to-hero-write-your-first-kubernetes-charm/study-your-application.md b/docs/tutorial/from-zero-to-hero-write-your-first-kubernetes-charm/study-your-application.md new file mode 100644 index 000000000..5b47de81e --- /dev/null +++ b/docs/tutorial/from-zero-to-hero-write-your-first-kubernetes-charm/study-your-application.md @@ -0,0 +1,105 @@ +(study-your-application)= +# Study your application + +> {ref}`From Zero to Hero: Write your first Kubernetes charm ` > Study your application + +A charm is an application packaged with all the logic it needs to operate in the cloud. + +As such, if you want to charm an application, the first thing you need to understand is the application itself. + +Of course, what exactly you'll need to know and how exactly you'll have to go about getting this knowledge will very much depend on the application. + +In this part of the tutorial we will choose an application for you and tell you all you need to know about it to start charming. Our demo app is called 'FastAPI Demo' and we have designed it specifically for this tutorial so that, by creating a Kubernetes charm for it, you can master all the fundamentals of Kubernetes charming. + + + + + + +## Features + +The FastAPI app was built using the Python [FastAPI](https://fastapi.tiangolo.com/) framework to deliver a very simple web server. It offers a couple of API endpoints that the user can interact with. + +The app also has a connection to a [PostgreSQL](https://www.postgresql.org/) database. It provides users with an API to create a table with user names, add a name to the database, and get all the names from the database. + +Additionally, our app uses [starlette-exporter](https://pypi.org/project/starlette-exporter/) to generate real-time application metrics and to expose them via a `/metrics` endpoint that is designed to be scraped by [Prometheus](https://prometheus.io/). + +Finally, every time a user interacts with the database, our app writes logging information to the log file and also streams it to the stdout. + +To summarize, our demo application is a minimal but real-life-like application that has external API endpoints, performs database read and write operations, and collects real-time metrics and logs for observability purposes. + + +## Structure + +The app source code is hosted at https://github.com/canonical/api_demo_server . + +As you can see [here](https://github.com/canonical/api_demo_server/tree/master/api_demo_server), the app consists of primarily the following two files: + +- `app.py`, which describes the API endpoints and the logging definition, and +- `database.py`, which describes the interaction with the PostgreSQL database. + +Furthermore, as you can see [here](https://github.com/canonical/api_demo_server/tree/master?tab=readme-ov-file#configuration-via-environment-variables), the application provides a way to configure the output logging file and the database access points (IP, port, username, password) via environment variables: + +- `DEMO_SERVER_LOGFILE` +- `DEMO_SERVER_DB_HOST` +- `DEMO_SERVER_DB_PORT` +- `DEMO_SERVER_DB_USER` +- `DEMO_SERVER_DB_PASSWORD` + + +## API endpoints + +The application is set up such that, once deployed, you can access the deployed IP on port 8000. Specifically: + + +||| +|-|-| +| To get Prometheus metrics: | `http://:8000/metrics` | +| To get a Swagger UI to interact with API: |`http://:8000/docs`| + + + +## OCI image + +Our app's OCI image is at + +https://github.com/canonical/api_demo_server/pkgs/container/api_demo_server + + + + +> **See next: {ref}`Set up your development environment `** + + diff --git a/docs/tutorial/from-zero-to-hero-write-your-first-kubernetes-charm/write-integration-tests-for-your-charm.md b/docs/tutorial/from-zero-to-hero-write-your-first-kubernetes-charm/write-integration-tests-for-your-charm.md new file mode 100644 index 000000000..0540e77c7 --- /dev/null +++ b/docs/tutorial/from-zero-to-hero-write-your-first-kubernetes-charm/write-integration-tests-for-your-charm.md @@ -0,0 +1,204 @@ +(write-integration-tests-for-your-charm)= +# Write integration tests for your charm + +> {ref}`From Zero to Hero: Write your first Kubernetes charm ` > Write integration tests for your charm +> +> **See previous: {ref}`Write scenario tests for your charm `** + +````{important} + +This document is part of a series, and we recommend you follow it in sequence. However, you can also jump straight in by checking out the code from the previous branches: + +```bash +git clone https://github.com/canonical/juju-sdk-tutorial-k8s.git +cd juju-sdk-tutorial-k8s +git checkout 09_scenario_test +git checkout -b 10_integration_testing +``` + +```` + +A charm should function correctly not just in a mocked environment but also in a real deployment. + +For example, it should be able to pack, deploy, and integrate without throwing exceptions or getting stuck in a `waiting` or a `blocked` status -- that is, it should correctly reach a status of `active` or `idle`. + +You can ensure this by writing integration tests for your charm. In the charming world, these are usually written with the [`pytest-operator`](https://github.com/charmed-kubernetes/pytest-operator) library. + +In this chapter you will write two small integration tests -- one to check that the charm packs and deploys correctly and one to check that the charm integrates successfully with the PostgreSQL database. + +## Prepare your test environment + +In your `tox.ini` file, add the following new environment: + +``` +[testenv:integration] +description = Run integration tests +deps = + pytest + juju + pytest-operator + -r {tox_root}/requirements.txt +commands = + pytest -v \ + -s \ + --tb native \ + --log-cli-level=INFO \ + {posargs} \ + {[vars]tests_path}/integration +``` + +## Prepare your test directory + +Create a `tests/integration` directory: +```bash +mkdir ~/fastapi-demo/tests/integration + +``` + +## Write and run a pack-and-deploy integration test + +Let's begin with the simplest possible integration test, a [smoke test](https://en.wikipedia.org/wiki/Smoke_testing_(software)). This test will build and deploy the charm and verify that the installation hooks finish without any error. + + +In your `tests/integration` directory, create a file `test_charm.py` and add the following test case: + +```python +import asyncio +import logging +from pathlib import Path + +import pytest +import yaml +from pytest_operator.plugin import OpsTest + +logger = logging.getLogger(__name__) + +METADATA = yaml.safe_load(Path('./charmcraft.yaml').read_text()) +APP_NAME = METADATA['name'] + + +@pytest.mark.abort_on_fail +async def test_build_and_deploy(ops_test: OpsTest): + """Build the charm-under-test and deploy it together with related charms. + + Assert on the unit status before any relations/configurations take place. + """ + # Build and deploy charm from local source folder + charm = await ops_test.build_charm('.') + resources = { + 'demo-server-image': METADATA['resources']['demo-server-image']['upstream-source'] + } + + # Deploy the charm and wait for blocked/idle status + # The app will not be in active status as this requires a database relation + await asyncio.gather( + ops_test.model.deploy(charm, resources=resources, application_name=APP_NAME), + ops_test.model.wait_for_idle( + apps=[APP_NAME], status='blocked', raise_on_blocked=False, timeout=120 + ), + ) +``` + +In your Multipass Ubuntu VM, run the test: + +```bash +tox -e integration +``` + +The test takes some time to run as the `pytest-operator` running in the background will add a new model to an existing cluster (whose presence it assumes). If successful, it'll verify that your charm can pack and deploy as expected. + +## Write and run an integrate-with-database integration test + +The charm requires a database to be functional. Let's verify that this behaviour works as intended. For that, we need to deploy a database to the test cluster and integrate both applications. Finally, we should check that the charm reports an active status. + +In your `tests/integration/test_charm.py` file add the following test case: + +```python +@pytest.mark.abort_on_fail +async def test_database_integration(ops_test: OpsTest): + """Verify that the charm integrates with the database. + + Assert that the charm is active if the integration is established. + """ + await ops_test.model.deploy( + application_name='postgresql-k8s', + entity_url='postgresql-k8s', + channel='14/stable', + ) + await ops_test.model.integrate(f'{APP_NAME}', 'postgresql-k8s') + await ops_test.model.wait_for_idle( + apps=[APP_NAME], status='active', raise_on_blocked=False, timeout=120 + ) +``` + + +```{important} + +But if you run the one and then the other (as separate `pytest ...` invocations, then two separate models will be created unless you pass `--model=some-existing-model` to inform pytest-operator to use a model you provide. + +``` + +In your Multipass Ubuntu VM, run the test again: + + +```bash +ubuntu@charm-dev:~/fastapi-demo$ tox -e integration + +``` + +The test may again take some time to run. + +```{tip} + +**Pro tip:** To make things faster, use the `--model=` to inform `pytest-operator` to use the model it has created for the first test. Otherwise, charmers often have a way to cache their pack or deploy results; an example is https://github.com/canonical/spellbook . + +``` + +When it's done, the output should show two passing tests: + +```bash +... + demo-api-charm/0 [idle] waiting: Waiting for database relation +INFO juju.model:model.py:2759 Waiting for model: + demo-api-charm/0 [idle] active: +PASSED +-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- live log teardown -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- +INFO pytest_operator.plugin:plugin.py:783 Model status: + +Model Controller Cloud/Region Version SLA Timestamp +test-charm-2ara main-controller microk8s/localhost 3.1.5 unsupported 09:45:56+02:00 + +App Version Status Scale Charm Channel Rev Address Exposed Message +demo-api-charm 1.0.1 active 1 demo-api-charm 0 10.152.183.99 no +postgresql-k8s 14.7 active 1 postgresql-k8s 14/stable 73 10.152.183.50 no + +Unit Workload Agent Address Ports Message +demo-api-charm/0* active idle 10.1.208.77 +postgresql-k8s/0* active idle 10.1.208.107 + +INFO pytest_operator.plugin:plugin.py:789 Juju error logs: + + +INFO pytest_operator.plugin:plugin.py:877 Resetting model test-charm-2ara... +INFO pytest_operator.plugin:plugin.py:866 Destroying applications demo-api-charm +INFO pytest_operator.plugin:plugin.py:866 Destroying applications postgresql-k8s +INFO pytest_operator.plugin:plugin.py:882 Not waiting on reset to complete. +INFO pytest_operator.plugin:plugin.py:855 Forgetting main... + + +========================================================================================================================================================================== 2 passed in 290.23s (0:04:50) ========================================================================================================================================================================== + integration: OK (291.01=setup[0.04]+cmd[290.97] seconds) + congratulations :) (291.05 seconds) +``` + +Congratulations, with this integration test you have verified that your charms relation to PostgreSQL works as well! + +## Review the final code + +For the full code see: [10_integration_testing](https://github.com/canonical/juju-sdk-tutorial-k8s/tree/09_scenario_test) + +For a comparative view of the code before and after this doc see: [Comparison](https://github.com/canonical/juju-sdk-tutorial-k8s/compare/09_scenario_test...10_integration_testing) + +> **See next: {ref}`Open a Kubernetes port in your charm `** + + diff --git a/docs/tutorial/from-zero-to-hero-write-your-first-kubernetes-charm/write-scenario-tests-for-your-charm.md b/docs/tutorial/from-zero-to-hero-write-your-first-kubernetes-charm/write-scenario-tests-for-your-charm.md new file mode 100644 index 000000000..c401c4e16 --- /dev/null +++ b/docs/tutorial/from-zero-to-hero-write-your-first-kubernetes-charm/write-scenario-tests-for-your-charm.md @@ -0,0 +1,204 @@ +(write-scenario-tests-for-your-charm)= +# Write scenario tests for your charm + +> {ref}`From Zero to Hero: Write your first Kubernetes charm ` > Write scenario tests for your charm +> +> **See previous: {ref}`Write unit tests for your charm `** + +````{important} + +This document is part of a series, and we recommend you follow it in sequence. However, you can also jump straight in by checking out the code from the previous branches: + +``` +git clone https://github.com/canonical/juju-sdk-tutorial-k8s.git +cd juju-sdk-tutorial-k8s +git checkout 08_unit_testing +git checkout -b 09_scenario_testing +``` + +```` + +In the previous chapter we checked the basic functionality of our charm by writing unit tests. + +However, there is one more type of test to cover, namely: state transition tests. + +In the charming world the current recommendation is to write state transition tests with the 'scenario' model popularised by the {ref}``ops-scenario` ` library. + +```{note} + Scenario is a state-transition testing SDK for operator framework charms. +``` + +In this chapter you will write a scenario test to check that the `get_db_info` action that you defined in an earlier chapter behaves as expected. + + +## Prepare your test environment + +Install `ops-scenario`: + +```bash +pip install ops-scenario +``` +In your project root's existing `tox.ini` file, add the following: + +``` +... + +[testenv:scenario] +description = Run scenario tests +deps = + pytest + cosl + ops-scenario ~= 7.0 + coverage[toml] + -r {tox_root}/requirements.txt +commands = + coverage run --source={[vars]src_path} \ + -m pytest \ + --tb native \ + -v \ + -s \ + {posargs} \ + {[vars]tests_path}/scenario + coverage report +``` + +And adjust the `env_list` so that the Scenario tests will run with a plain `tox` command: + +``` +env_list = unit, scenario +``` + +## Prepare your test directory + +By convention, scenario tests are kept in a separate directory, `tests/scenario`. Create it as below: + +``` +mkdir -p tests/scenario +cd tests/scenario +``` + + +## Write your scenario test + +In your `tests/scenario` directory, create a new file `test_charm.py` and add the test below. This test will check the behaviour of the `get_db_info` action that you set up in a previous chapter. It will first set up the test context by setting the appropriate metadata, then define the input state, then run the action and, finally, check if the results match the expected values. + +```python +from unittest.mock import Mock + +import scenario +from pytest import MonkeyPatch + +from charm import FastAPIDemoCharm + + +def test_get_db_info_action(monkeypatch: MonkeyPatch): + monkeypatch.setattr('charm.LogProxyConsumer', Mock()) + monkeypatch.setattr('charm.MetricsEndpointProvider', Mock()) + monkeypatch.setattr('charm.GrafanaDashboardProvider', Mock()) + + # Use scenario.Context to declare what charm we are testing. + # Note that Scenario will automatically pick up the metadata from + # your charmcraft.yaml file, so you typically could just do + # `ctx = scenario.Context(FastAPIDemoCharm)` here, but the full + # version is included here as an example. + ctx = scenario.Context( + FastAPIDemoCharm, + meta={ + 'name': 'demo-api-charm', + 'containers': {'demo-server': {}}, + 'peers': {'fastapi-peer': {'interface': 'fastapi_demo_peers'}}, + 'requires': { + 'database': { + 'interface': 'postgresql_client', + } + }, + }, + config={ + 'options': { + 'server-port': { + 'default': 8000, + } + } + }, + actions={ + 'get-db-info': {'params': {'show-password': {'default': False, 'type': 'boolean'}}} + }, + ) + + # Declare the input state. + state_in = scenario.State( + leader=True, + relations={ + scenario.Relation( + endpoint='database', + interface='postgresql_client', + remote_app_name='postgresql-k8s', + local_unit_data={}, + remote_app_data={ + 'endpoints': '127.0.0.1:5432', + 'username': 'foo', + 'password': 'bar', + }, + ), + }, + containers={ + scenario.Container('demo-server', can_connect=True), + }, + ) + + # Run the action with the defined state and collect the output. + ctx.run(ctx.on.action('get-db-info', params={'show-password': True}), state_in) + + assert ctx.action_results == { + 'db-host': '127.0.0.1', + 'db-port': '5432', + 'db-username': 'foo', + 'db-password': 'bar', + } +``` + + +## Run the test + +In your Multipass Ubuntu VM shell, run your scenario test as below: + +```bash +ubuntu@charm-dev:~/juju-sdk-tutorial-k8s$ tox -e scenario +``` + +You should get an output similar to the one below: + +```bash +scenario: commands[0]> coverage run --source=/home/tameyer/code/juju-sdk-tutorial-k8s/src -m pytest --tb native -v -s /home/tameyer/code/juju-sdk-tutorial-k8s/tests/scenario +======================================= test session starts ======================================== +platform linux -- Python 3.11.9, pytest-8.3.3, pluggy-1.5.0 -- /home/tameyer/code/juju-sdk-tutorial-k8s/.tox/scenario/bin/python +cachedir: .tox/scenario/.pytest_cache +rootdir: /home/tameyer/code/juju-sdk-tutorial-k8s +plugins: anyio-4.6.0 +collected 1 item + +tests/scenario/test_charm.py::test_get_db_info_action PASSED + +======================================== 1 passed in 0.19s ========================================= +scenario: commands[1]> coverage report +Name Stmts Miss Cover +---------------------------------- +src/charm.py 129 57 56% +---------------------------------- +TOTAL 129 57 56% + scenario: OK (6.89=setup[6.39]+cmd[0.44,0.06] seconds) + congratulations :) (6.94 seconds) +``` + +Congratulations, you have written your first scenario test! + +## Review the final code + + +For the full code see: [09_scenario_testing](https://github.com/canonical/juju-sdk-tutorial-k8s/tree/09_scenario_test) + +For a comparative view of the code before and after this doc see: [Comparison](https://github.com/canonical/juju-sdk-tutorial-k8s/compare/08_unit_testing...09_scenario_test) + +> **See next: {ref}`Write integration tests for your charm `** + + diff --git a/docs/tutorial/from-zero-to-hero-write-your-first-kubernetes-charm/write-unit-tests-for-your-charm.md b/docs/tutorial/from-zero-to-hero-write-your-first-kubernetes-charm/write-unit-tests-for-your-charm.md new file mode 100644 index 000000000..2f9ff9cc4 --- /dev/null +++ b/docs/tutorial/from-zero-to-hero-write-your-first-kubernetes-charm/write-unit-tests-for-your-charm.md @@ -0,0 +1,200 @@ +(write-unit-tests-for-your-charm)= +# Write unit tests for your charm + +> {ref}`From Zero to Hero: Write your first Kubernetes charm ` > Write a unit test for your charm +> +> **See previous: {ref}`Observe your charm with COS Lite `** + +````{important} + +This document is part of a series, and we recommend you follow it in sequence. However, you can also jump straight in by checking out the code from the previous branches: + +```bash +git clone https://github.com/canonical/juju-sdk-tutorial-k8s.git +cd juju-sdk-tutorial-k8s +git checkout 07_cos_integration +git checkout -b 08_unit_testing +``` + +```` + +When you're writing a charm, you will want to ensure that it will behave reliably as intended. + +For example, that the various components -- relation data, pebble services, or configuration files -- all behave as expected in response to an event. + +You can ensure all this by writing a rich battery of units tests. In the context of a charm we recommended using [`pytest`](https://pytest.org/) (but [`unittest`](https://docs.python.org/3/library/unittest.html) can also be used) and especially the operator framework's built-in testing library -- [`ops.testing.Harness`](https://ops.readthedocs.io/en/latest/harness.html#module-ops.testing). We will be using the Python testing tool [`tox`](https://tox.wiki/en/4.14.2/index.html) to automate our testing and set up our testing environment. + +In this chapter you will write a simple unit test to check that your workload container is initialised correctly. + + +## Prepare your test environment + +Create a file called `tox.ini` in your charm project's root directory and add the following to configure your test environment: + +``` +[tox] +no_package = True +skip_missing_interpreters = True +min_version = 4.0.0 +env_list = unit + +[vars] +src_path = {tox_root}/src +tests_path = {tox_root}/tests + +[testenv] +set_env = + PYTHONPATH = {tox_root}/lib:{[vars]src_path} + PYTHONBREAKPOINT=pdb.set_trace + PY_COLORS=1 +pass_env = + PYTHONPATH + CHARM_BUILD_DIR + MODEL_SETTINGS + +[testenv:unit] +description = Run unit tests +deps = + pytest + cosl + coverage[toml] + -r {tox_root}/requirements.txt +commands = + coverage run --source={[vars]src_path} \ + -m pytest \ + --tb native \ + -v \ + -s \ + {posargs} \ + {[vars]tests_path}/unit + coverage report +``` +> Read more: [`tox.ini`](https://tox.wiki/en/latest/config.html#tox-ini) + + +## Prepare your test directory + +In your project root, create a `tests/unit` directory: + +```bash +mkdir -p tests/unit +``` + +### Write your unit test + +In your `tests/unit` directory, create a file called `test_charm.py`. + +In this file, do all of the following: + +First, add the necessary imports: + +```python +import ops +import ops.testing +import pytest + +from charm import FastAPIDemoCharm +``` + +Then, add a test [fixture](https://docs.pytest.org/en/7.1.x/how-to/fixtures.html) that sets up the testing harness and makes sure that it will be cleaned up after each test: + +```python +@pytest.fixture +def harness(): + harness = ops.testing.Harness(FastAPIDemoCharm) + harness.begin() + yield harness + harness.cleanup() + +``` + +Finally, add a first test case as a function, as below. As you can see, this test case is used to verify that the deployment of the `fastapi-service` within the `demo-server` container is configured correctly and that the service is started and running as expected when the container is marked as `pebble-ready`. It also checks that the unit's status is set to active without any error messages. Note that we mock some methods of the charm because they do external calls that are not represented in the state of this unit test. + +```python +def test_pebble_layer( + monkeypatch: pytest.MonkeyPatch, harness: ops.testing.Harness[FastAPIDemoCharm] +): + monkeypatch.setattr(FastAPIDemoCharm, 'version', '1.0.0') + # Expected plan after Pebble ready with default config + expected_plan = { + 'services': { + 'fastapi-service': { + 'override': 'replace', + 'summary': 'fastapi demo', + 'command': 'uvicorn api_demo_server.app:app --host=0.0.0.0 --port=8000', + 'startup': 'enabled', + # Since the environment is empty, Layer.to_dict() will not + # include it. + } + } + } + + # Simulate the container coming up and emission of pebble-ready event + harness.container_pebble_ready('demo-server') + harness.evaluate_status() + + # Get the plan now we've run PebbleReady + updated_plan = harness.get_container_pebble_plan('demo-server').to_dict() + service = harness.model.unit.get_container('demo-server').get_service('fastapi-service') + # Check that we have the plan we expected: + assert updated_plan == expected_plan + # Check the service was started: + assert service.is_running() + # Ensure we set a BlockedStatus with appropriate message: + assert isinstance(harness.model.unit.status, ops.BlockedStatus) + assert 'Waiting for database' in harness.model.unit.status.message +``` + + +> Read more: [`ops.testing`](https://ops.readthedocs.io/en/latest/harness.html#module-ops.testing) + +## Run the test + +In your Multipass Ubuntu VM shell, run your unit test as below: + +```bash +ubuntu@charm-dev:~/fastapi-demo$ tox -e unit +``` + +You should get an output similar to the one below: + +```bash +unit: commands[0]> coverage run --source=/home/ubuntu/fastapi-demo/src -m pytest --tb native -v -s /home/ubuntu/fastapi-demo/tests/unit +=============================================================================================================================================================================== test session starts =============================================================================================================================================================================== +platform linux -- Python 3.10.13, pytest-8.0.2, pluggy-1.4.0 -- /home/ubuntu/fastapi-demo/.tox/unit/bin/python +cachedir: .tox/unit/.pytest_cache +rootdir: /home/ubuntu/fastapi-demo +collected 1 item + +tests/unit/test_charm.py::test_pebble_layer PASSED + +================================================================================================================================================================================ 1 passed in 0.30s ================================================================================================================================================================================ +unit: commands[1]> coverage report +Name Stmts Miss Cover +---------------------------------- +src/charm.py 118 49 58% +---------------------------------- +TOTAL 118 49 58% + unit: OK (0.99=setup[0.04]+cmd[0.78,0.16] seconds) + congratulations :) (1.02 seconds) +``` + +Congratulations, you have now successfully implemented your first unit test! + +```{caution} + +As you can see in the output, the current tests cover 58% of the charm code. In a real-life scenario make sure to cover much more! + +``` + +## Review the final code + +For the full code see: [08_unit_testing](https://github.com/canonical/juju-sdk-tutorial-k8s/tree/08_unit_testing) + +For a comparative view of the code before and after this doc see: [Comparison](https://github.com/canonical/juju-sdk-tutorial-k8s/compare/07_cos_integration...08_unit_testing) + +> **See next: {ref}`Write scenario tests for your charm `** + + + + diff --git a/docs/tutorial/index.md b/docs/tutorial/index.md new file mode 100644 index 000000000..bef71fe7b --- /dev/null +++ b/docs/tutorial/index.md @@ -0,0 +1,12 @@ +(tutorial)= +# Tutorials + +Writing a machine charm is not so different from writing a Kubernetes charm, but it *is* a little bit different. As such, our tutorial comes in two basic flavors, for machines and for Kubernetes. The choice is yours! + + +```{toctree} +:maxdepth: 1 + +write-your-first-machine-charm +from-zero-to-hero-write-your-first-kubernetes-charm/index +``` diff --git a/docs/tutorial/write-your-first-machine-charm.md b/docs/tutorial/write-your-first-machine-charm.md new file mode 100644 index 000000000..d3e8da698 --- /dev/null +++ b/docs/tutorial/write-your-first-machine-charm.md @@ -0,0 +1,530 @@ +(write-your-first-machine-charm)= +# Write your first machine charm + +In this tutorial you will write a [machine charm](https://juju.is/docs/juju/charmed-operator) for [`juju`](https://juju.is/docs/juju) using (`charmcraft` and) `ops`. + + +**What you'll need:** +- A workstation, e.g., a laptop, with amd64 architecture and which has sufficient resources to launch a virtual machine with 4 CPUs, 8 GB RAM, and 50 GB disk space +- Familiarity with Linux +- Familiarity with [Juju](https://juju.is/docs/juju) +- Familiarity with object-oriented programming in Python + +**What you'll do:** + +Study your application. Use `charmcraft` and `ops` to build a basic charm and test-deploy it with `juju` and a localhost LXD-based cloud. Repeat the steps to evolve the charm so it can become increasingly more sophisticated. + + + +```{note} + +Should you get stuck at any point: Don't hesitate to get in touch on [Matrix](https://matrix.to/#/#charmhub-charmdev:ubuntu.com) or [Discourse](https://discourse.charmhub.io/). + +``` + +## Study your application + +In this tutorial we will be writing a charm for Microsample (`microsample`) -- a small educational application that delivers a Flask microservice. + +The application has been packaged and published as a snap ([https://snapcraft.io/microsample](https://snapcraft.io/microsample)). We will write our charm such that `juju deploy` will install it from this snap. This will make workload installation straightforward and upgrades automatic (as they will happen automatically through `snapd`). + +The application snap has been released into multiple channels -- `edge`, `beta`, `candidate`, and `stable`. We will write our charm such that a user can choose the channel they prefer by running `juju deploy microsample channel=`. + +The application has other features that we can exploit, but for now this is enough to get us started with a simple charm. + + +## Set up your development environment + +> See [`juju` | Set up your development environment automatically](https://juju.is/docs/juju/set-up--tear-down-your-test-environment#set-up-automatically) for instructions on how to set up your development environment so that it's ready for you to test-deploy your charm. At the charm directory step, call it `microsample-vm`. At the cloud step, choose LXD. + +```{important} + +- Going forward: + - Use your host machine (on Linux, `cd ~/microsample-vm`) to create and edit your charm files. This will allow you to use your favorite local editor. + - Use the Multipass VM shell (on Linux, `ubuntu@charm-dev:~$ cd ~/microsample-vm`) to run Charmcraft and Juju commands. + + +- At any point: + - To exit the shell, press `mod key + C` or type `exit`. + - To stop the VM after exiting the VM shell, run `multipass stop charm-dev`. + - To restart the VM and re-open a shell into it, type `multipass shell charm-dev`. + +``` + + +## Enable `juju deploy microsample-vm` + + +Let's charm our `microsample` application into a `microsample-vm` charm such that a user can successfully install it on any machine cloud simply by running `juju deploy microsample-vm`! + +In your Multipass VM shell, enter your charm directory, run `charmcraft init --profile machine` to initialise the file tree structure for your machine charm, and inspect the result. Sample session: + +```text +# Enter your charm directory: +ubuntu@charm-dev:~$ cd microsample-vm/ + +# Initialise the charm tree structure: +ubuntu@charm-dev:~/microsample-vm$ charmcraft init --profile machine +Charmed operator package file and directory tree initialised. + +Now edit the following package files to provide fundamental charm metadata +and other information: + +charmcraft.yaml +src/charm.py +README.md + +# Inspect the result: +ubuntu@charm-dev:~/microsample-vm$ ls -R +.: +CONTRIBUTING.md README.md pyproject.toml src tox.ini +LICENSE charmcraft.yaml requirements.txt tests + +./src: +charm.py + +./tests: +integration unit + +./tests/integration: +test_charm.py + +./tests/unit: +test_charm.py + +``` + +> See more: [`charmcraft` | Manage charms](), [`charmcraft` | List of files in a charm project]() + +In your local editor, open the `charmcraft.yaml` file and customise its contents as below (you only have to edit the `title`, `summary`, and `description`): + +```yaml +# (Required) +name: microsample-vm + +# (Required) +type: charm + +# (Recommended) +title: Microsample VM Charm + +# (Required) +summary: A charm that deploys the microsample snap and allows for a configuration of the snap channel via juju config. + +# (Required) +description: | + A machine charm for the Microsample application, built on top of the `microsample` snap. + + The charm allows you to deploy the application via `juju deploy`. + It also defines a channel config that allows you to choose which snap channel to install from during deployment. + + This charm makes it easy to deploy the Microsample application on any machine cloud. + + The primary value of this charm is educational -- beginner machine charms can study it to learn how to build a machine charm. + +# (Required for 'charm' type) +bases: + - build-on: + - name: ubuntu + channel: "22.04" + run-on: + - name: ubuntu + channel: "22.04" + +``` + +> See more: [`charmcraft` | File `charmcraft.yaml`]() + +Now open the `src/charm.py` file and update it as below (you'll have to add an import statement for `os` and an observer and handler for the `install` event -- in the definition of which you'll be using `os` and `ops`). + +```python +#!/usr/bin/env python3 +import os +import logging +import ops + +logger = logging.getLogger(__name__) + +class MicrosampleVmCharm(ops.CharmBase): + + def __init__(self, *args): + super().__init__(*args) + self.framework.observe(self.on.start, self._on_start) + self.framework.observe(self.on.install, self._on_install) + + def _on_start(self, event: ops.StartEvent): + """Handle start event.""" + self.unit.status = ops.ActiveStatus() + + def _on_install(self, event: ops.InstallEvent): + """Handle install event.""" + self.unit.status = ops.MaintenanceStatus("Installing microsample snap") + os.system(f"snap install microsample --channel edge") + self.unit.status = ops.ActiveStatus("Ready") + + +if __name__ == "__main__": # pragma: nocover + ops.main(MicrosampleVmCharm) # type: ignore +``` + +> See more: [`charmcraft` | File `src/charm.py`](), {ref}`run-workloads-with-a-charm-machines` + +Next, in your Multipass VM shell, inside your project directory, run `charmcraft pack` to pack the charm. It may take a few minutes the first time around but, when it's done, your charm project should contain a `.charm` file. Sample session: + + +```text +# Pack the charm into a '.charm' file: +ubuntu@charm-dev:~/microsample-vm$ charmcraft pack +Created 'microsample-vm_ubuntu-22.04-amd64.charm'. +Charms packed: + microsample-vm_ubuntu-22.04-amd64.charm + +# Inspect the results -- your charm's root directory should contain a .charm file: +ubuntu@charm-dev:~/microsample-vm$ ls +CONTRIBUTING.md charmcraft.yaml requirements.txt tox.ini +LICENSE microsample-vm_ubuntu-22.04-amd64.charm src +README.md pyproject.toml tests +``` + +> See more: [`charmcraft` | Manage charms > Pack]() + +Now, open a new shell into your Multipass VM and use it to configure the Juju log verbosity levels and to start a live debug session: + +```text +# Set your logging verbosity level to `DEBUG`: +ubuntu@charm-dev:~$ juju model-config logging-config="=WARNING;unit=DEBUG" + +# Start a live debug session: +ubuntu@charm-dev:~$ juju debug-log +``` + +In your old VM shell, use Juju to deploy your charm. If all has gone well, you should see your App and Unit -- Workload status show as `active`: + +```text +# Deploy the Microsample VM charm as the 'microsample' application: +ubuntu@charm-dev:~/microsample-vm$ juju deploy ./microsample-vm_ubuntu-22.04-amd64.charm microsample +Located local charm "microsample-vm", revision 0 +Deploying "microsample" from local charm "microsample-vm", revision 0 on ubuntu@22.04/stable + +# Check the deployment status +# (use --watch 1s to update it automatically at 1s intervals): +ubuntu@charm-dev:~/microsample-vm$ juju status +Model Controller Cloud/Region Version SLA Timestamp +welcome-lxd lxd localhost/localhost 3.1.6 unsupported 12:49:26+01:00 + +App Version Status Scale Charm Channel Rev Exposed Message +microsample active 1 microsample-vm 0 no + +Unit Workload Agent Machine Public address Ports Message +microsample/0* active idle 1 10.122.219.101 + +Machine State Address Inst id Base AZ Message +1 started 10.122.219.101 juju-f25b73-1 ubuntu@22.04 Running + + +``` + +Finally, test that the service works by executing `curl` on your application unit: + +```text +ubuntu@charm-dev:~/microsample-vm$ juju exec --unit microsample/0 -- "curl -s http://localhost:8080" +Online +``` + +```{note} + +1. Fix the code in `src/charm.py`. +2. Rebuild the charm: `charmcraft pack` +3. Refresh the application from the repacked charm: `juju refresh microsample --path=./microsample-vm_ubuntu-22.04-amd64.charm --force-units` +4. Let the model know the issue is resolved (fixed): `juju resolved microsample/0`. + +``` + + + + + +```{note} + +The template content from `charmcraft init` was sufficient for the charm to pack and deploy successfully. However, our goal here was to make it run successfully, that is, to actually install the `microsample` application on our LXD cloud. With the edits above, this goal has been achieved. + +``` + + +## Enable `juju deploy microsample-vm --config channel=` + +Let's now evolve our charm so that a user can successfully choose which version of `microsample` they want installed by running `juju config microsample-vm channel=`! + +In your local editor, in your `charmcraft.yaml` file, define the configuration option as below: + +```yaml +config: + options: + channel: + description: | + Channel for the microsample snap. + default: "edge" + type: string +``` + +> See more: [`charmcraft` | File `charmcraft.yaml` > Key `config`]() + +Then, in the `src/charm.py` file, update the `_on_install` function to make use of the new configuration option, as below: + +```python +def _on_install(self, event: ops.ConfigChangedEvent): + """Handle install event.""" + self.unit.status = ops.MaintenanceStatus("Installing microsample snap") + channel = self.config.get('channel') + if channel in {ref}`'beta', 'edge', 'candidate', 'stable']: + os.system(f"snap install microsample --{channel}") + self.unit.status = ops.ActiveStatus("Ready") + else: + self.unit.status = ops.BlockedStatus("Invalid channel configured.") +``` + +Now, in your Multipass VM shell, inside your project directory, pack the charm, refresh it in the Juju model, and inspect the results: + +```text + +# Pack the charm: +ubuntu@charm-dev:~/microsample-vm$ charmcraft pack +Created 'microsample-vm_ubuntu-22.04-amd64.charm'. +Charms packed: + microsample-vm_ubuntu-22.04-amd64.charm + +# Refresh the application from the repacked charm: +ubuntu@charm-dev:~/microsample-vm$ juju refresh microsample --path=./microsample-vm_ubuntu-22.04-amd64.charm +Added local charm "microsample-vm", revision 1, to the model + +# Verify that the new configuration option is available: +ubuntu@charm-dev:~/microsample-vm$ juju config microsample +application: microsample +application-config: + trust: + default: false + description: Does this application have access to trusted credentials + source: default + type: bool + value: false +charm: microsample-vm +settings: + channel: + default: edge + description: | + Channel for the microsample snap. + source: default + type: string + value: edge + +``` + +Back to the `src/charm.py` file, in the `__init__` function of your charm, observe the `config-changed` event and pair it with an event handler: + +```text +self.framework.observe(self.on.config_changed, self._on_config_changed) +``` + + +Next, in the body of the charm definition, define the event handler, as below: + +```python +def _on_config_changed(self, event: ops.ConfigChangedEvent): + channel = self.config.get('channel') + if channel in {ref}`'beta', 'edge', 'candidate', 'stable']: + os.system(f"snap refresh microsample --{channel}") + self.unit.status = ops.ActiveStatus("Ready at '%s'" % channel) + else: + self.unit.status = ops.BlockedStatus("Invalid channel configured.") +``` + +Now, in your Multipass VM shell, inside your project directory, pack the charm, refresh it in the Juju model, and inspect the results: + +```text +# Pack the charm: +ubuntu@charm-dev:~/microsample-vm$ charmcraft pack +Created 'microsample-vm_ubuntu-22.04-amd64.charm'. +Charms packed: + microsample-vm_ubuntu-22.04-amd64.charm + +# Refresh the application: +ubuntu@charm-dev:~/microsample-vm$ juju refresh microsample --path=./microsample-vm_ubuntu-22.04-amd64.charm +Added local charm "microsample-vm", revision 2, to the model + +# Change the 'channel' config to 'beta': +ubuntu@charm-dev:~/microsample-vm$ juju config microsample channel=beta + +# Inspect the Message column +# ('Ready at beta' is what we expect to see if the snap channel has been changed to 'beta'): +ubuntu@charm-dev:~/microsample-vm$ juju status +Model Controller Cloud/Region Version SLA Timestamp +welcome-lxd lxd localhost/localhost 3.1.6 unsupported 13:54:53+01:00 + +App Version Status Scale Charm Channel Rev Exposed Message +microsample active 1 microsample-vm 2 no Ready at 'beta' + +Unit Workload Agent Machine Public address Ports Message +microsample/0* active idle 1 10.122.219.101 Ready at 'beta' + +Machine State Address Inst id Base AZ Message +1 started 10.122.219.101 juju-f25b73-1 ubuntu@22.04 Running +``` + +Congratulations, your charm users can now deploy the application from a specific channel! + +> See more: {ref}`manage-configurations` + + +## Enable `juju status` with `App Version` + +Let's evolve our charm so that a user can see which version of the application has been installed simply by running `juju status`! + +In your local editor, update the `requirements.txt` file as below (you'll have to add the `requests` and `requests-unixsocket` lines): + +```text +ops ~= 2.5 +requests==2.28.1 +requests-unixsocket==0.3.0 +``` + +> See more: [`charmcraft` | File `requirements.txt` `](), [PyPI > Library `requests`](https://pypi.org/project/requests/), [PyPI > Library `requests-unixsocket`](https://pypi.org/project/requests-unixsocket/) + +Then, in your `src/charm.py` file, import the `requests_unixsocket` package, update the `_on_config_changed` function to set the workload version to the output of a function `_getWorkloadVersion`, and define the function to retrieve the Microsample workload version from the `snapd` API via a Unix socket, as below: + +```python +#!/usr/bin/env python3 +# Copyright 2023 Ubuntu +# See LICENSE file for licensing details. + +"""Charm the application.""" + +import os +import logging +import ops +import requests_unixsocket + +logger = logging.getLogger(__name__) + + +class MicrosampleVmCharm(ops.CharmBase): + """Charm the application.""" + + def __init__(self, *args): + super().__init__(*args) + self.framework.observe(self.on.start, self._on_start) + self.framework.observe(self.on.install, self._on_install) + self.framework.observe(self.on.config_changed, self._on_config_changed) + + def _on_start(self, event: ops.StartEvent): + """Handle start event.""" + self.unit.status = ops.ActiveStatus() + + def _on_install(self, event: ops.InstallEvent): + """Handle install event.""" + self.unit.status = ops.MaintenanceStatus("Installing microsample snap") + channel = self.config.get('channel') + if channel in {ref}`'beta', 'edge', 'candidate', 'stable']: + os.system(f"snap install microsample --{channel}") + self.unit.status = ops.ActiveStatus("Ready") + else: + self.unit.status = ops.BlockedStatus("Invalid channel configured.") + + def _on_config_changed(self, event: ops.ConfigChangedEvent): + channel = self.config.get('channel') + if channel in ['beta', 'edge', 'candidate', 'stable']: + os.system(f"snap refresh microsample --{channel}") + workload_version = self._getWorkloadVersion() + self.unit.set_workload_version(workload_version) + self.unit.status = ops.ActiveStatus("Ready at '%s'" % channel) + else: + self.unit.status = ops.BlockedStatus("Invalid channel configured.") + + def _getWorkloadVersion(self): + """Get the microsample workload version from the snapd API via unix-socket""" + snap_name = "microsample" + snapd_url = f"http+unix://%2Frun%2Fsnapd.socket/v2/snaps/{snap_name}" + session = requests_unixsocket.Session() + # Use the requests library to send a GET request over the Unix domain socket + response = session.get(snapd_url) + # Check if the request was successful + if response.status_code == 200: + data = response.json() + workload_version = data["result"]["version"] + else: + workload_version = "unknown" + print(f"Failed to retrieve Snap apps. Status code: {response.status_code}") + + # Return the workload version + return workload_version + +if __name__ == "__main__": # pragma: nocover + ops.main(MicrosampleVmCharm) # type: ignore +``` + + + +Finally, in your Multipass VM shell, pack the charm, refresh it in Juju, and check the Juju status -- it should now show the version of your workload. + +```text +# Pack the charm: +ubuntu@charm-dev:~/microsample-vm$ charmcraft pack +Created 'microsample-vm_ubuntu-22.04-amd64.charm'. +Charms packed: + microsample-vm_ubuntu-22.04-amd64.charm + +# Refresh the application: +ubuntu@charm-dev:~/microsample-vm$ juju refresh microsample --path=./microsample-vm_ubuntu-22.04-amd64.charm +Added local charm "microsample-vm", revision 3, to the model + +# Verify that the App Version now shows the version: +ubuntu@charm-dev:~/microsample-vm$ juju status +Model Controller Cloud/Region Version SLA Timestamp +welcome-lxd lxd localhost/localhost 3.1.6 unsupported 14:04:39+01:00 + +App Version Status Scale Charm Channel Rev Exposed Message +microsample 0+git.49ff7aa active 1 microsample-vm 3 no Ready at 'beta' + +Unit Workload Agent Machine Public address Ports Message +microsample/0* active idle 1 10.122.219.101 Ready at 'beta' + +Machine State Address Inst id Base AZ Message +1 started 10.122.219.101 juju-f25b73-1 ubuntu@22.04 Running + + +``` + +Congratulations, your charm user can view the version of the workload deployed from your charm! + + +## Tear things down + +> See [`juju` | Tear down your development environment automatically](https://juju.is/docs/juju/set-up--tear-down-your-test-environment#tear-down-automatically) + +(tutorial-machines-next-steps)= +## Next steps + +By the end of this tutorial you will have built a machine charm and evolved it in a number of typical ways. But there is a lot more to explore: + +| If you are wondering... | visit... | +|-------------------------|----------------------| +| "How do I...?" | {ref}`how-to-guides` | +| "What is...?" | {ref}`reference` | +| "Why...?", "So what?" | {ref}`explanation` | + + From af370f50933b666c621b3bd7b0c375cff4ae1574 Mon Sep 17 00:00:00 2001 From: Ben Hoyt Date: Wed, 4 Dec 2024 09:49:28 +1300 Subject: [PATCH 02/16] Temporarily remove -W (error on warnings) so we can preview results --- tox.ini | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tox.ini b/tox.ini index ef9ee7f90..43442c04e 100644 --- a/tox.ini +++ b/tox.ini @@ -47,7 +47,9 @@ deps = pip-tools commands_pre = pip-sync {toxinidir}/docs/requirements.txt commands = - sphinx-build -W --keep-going docs/ docs/_build/html +# Temporarily remove -W (error on warnings) so we can preview results +# sphinx-build -W --keep-going docs/ docs/_build/html + sphinx-build --keep-going docs/ docs/_build/html [testenv:docs-live] description = Live development: build the Sphinx docs with autoreloading enabled From 680613f5442841f137c0fa47a76fb1c32240074a Mon Sep 17 00:00:00 2001 From: Ben Hoyt Date: Wed, 4 Dec 2024 09:57:31 +1300 Subject: [PATCH 03/16] Remove fail_on_warning for ReadTheDocs too --- .readthedocs.yaml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.readthedocs.yaml b/.readthedocs.yaml index 2edd6f79d..e8a02993d 100644 --- a/.readthedocs.yaml +++ b/.readthedocs.yaml @@ -14,4 +14,5 @@ build: sphinx: configuration: docs/conf.py - fail_on_warning: true +# Temporarily remove -W (error on warnings) so we can preview results +# fail_on_warning: true From 5ad155dd32c93c42c8f1048fa9ce8bab0eabb0c8 Mon Sep 17 00:00:00 2001 From: Tony Meyer Date: Wed, 4 Dec 2024 10:39:51 +1300 Subject: [PATCH 04/16] Resolve many warnings (writing output). There are many 'cross-reference target not found' warnings where there is no xref - these are all to content that isn't yet published - see the PR description for details. There is still a huge number of warnings in the earlier stages - I'll do a separate commit for those. --- docs/howto/get-started-with-charm-testing.md | 8 ++-- ...ument-your-charm-with-tracing-telemetry.md | 6 +-- docs/howto/manage-actions.md | 8 ++-- docs/howto/manage-interfaces.md | 8 ++-- docs/howto/manage-leadership-changes.md | 2 +- docs/howto/manage-storage.md | 4 +- docs/howto/manage-the-workload-version.md | 2 +- .../run-workloads-with-a-charm-kubernetes.md | 10 ++--- .../write-integration-tests-for-a-charm.md | 36 +++++++++--------- .../howto/write-scenario-tests-for-a-charm.md | 8 ++-- docs/ops.rst | 12 +++--- .../create_a_minimal_kubernetes_charm.jpg | Bin 0 -> 75153 bytes .../integrate_your_charm_with_postgresql.png | Bin 0 -> 19035 bytes .../observe_your_charm_with_cos_lite.png | Bin 0 -> 45611 bytes .../create-a-minimal-kubernetes-charm.md | 6 +-- .../integrate-your-charm-with-postgresql.md | 5 ++- .../observe-your-charm-with-cos-lite.md | 4 +- .../preserve-your-charms-data.md | 4 +- .../publish-your-charm-on-charmhub.md | 4 +- .../write-your-first-machine-charm.md | 14 +++---- 20 files changed, 70 insertions(+), 71 deletions(-) create mode 100644 docs/resources/create_a_minimal_kubernetes_charm.jpg create mode 100644 docs/resources/integrate_your_charm_with_postgresql.png create mode 100644 docs/resources/observe_your_charm_with_cos_lite.png diff --git a/docs/howto/get-started-with-charm-testing.md b/docs/howto/get-started-with-charm-testing.md index d947fc668..0c2edf63f 100644 --- a/docs/howto/get-started-with-charm-testing.md +++ b/docs/howto/get-started-with-charm-testing.md @@ -6,7 +6,7 @@ Testing charm code is an essential part of charming. Here we will see how to get **What you'll need:** - knowledge of testing in general - knowledge of Juju and charms -- knowledge of the Juju models and events, esp. the data involved in a charm's lifecycle (e.g. see {ref}`Talking to a workload control flow from A to Z `) +- knowledge of the Juju models and events, esp. the data involved in a charm's lifecycle (e.g. see [Talking to a workload control flow from A to Z]()) **What you will learn:** - What are the starting points for adding tests to a charm? @@ -77,7 +77,7 @@ If the charm is a machine charm, workload operation calls can be done directly, "Juju operations" are the most 'meta' of them all: they do not affect the workload in and of itself, but they share data which is meant to affect the operation of *other* charms that this charm is integrated with. -> See more: {ref}`Talking to a workload: control flow from A to Z `, {ref}`Charm lifecycle ` +> See more: [Talking to a workload: control flow from A to Z](), [Charm lifecycle]() ### What we are testing when we unit-test @@ -120,7 +120,7 @@ There are two ways to initialize a harnessed charm: * When a charm is deployed, it goes through the Setup phase, a fixed sequence of events. `Harness` has a method, `begin_with_initial_hooks()`, that runs this sequence. * Alternatively, you can initialise the charm by calling `begin()`. This will instantiate the charm without firing any Setup phase event. -> See more: {ref}`A charm's life `, [`ops.testing.Harness.begin_with_initial_hooks()`](https://ops.readthedocs.io/en/latest/harness.html#ops.testing.Harness.begin_with_initial_hooks), [`ops.testing.Harness.begin()`](https://ops.readthedocs.io/en/latest/harness.html#ops.testing.Harness.begin) +> See more: [A charm's life](), [`ops.testing.Harness.begin_with_initial_hooks()`](https://ops.readthedocs.io/en/latest/harness.html#ops.testing.Harness.begin_with_initial_hooks), [`ops.testing.Harness.begin()`](https://ops.readthedocs.io/en/latest/harness.html#ops.testing.Harness.begin) After the Setup phase, the charm goes into Operation. To test operation-phase-related events, the harness provides some methods to simulate the most common scenarios. For example: @@ -256,7 +256,7 @@ A good integration testing suite will check that the charm continues to operate Some charms represent their workload by means of an object-oriented wrapper, which mediates between operator code and the implementation of operation logic. In such cases, it can be useful to add a third category of tests, namely functional tests, that black-box test that workload wrapper without worrying about the substrate it runs on (the charm, the cloud, the machine or pod...). For an example charm adopting this strategy, see [parca-operator](https://github.com/jnsgruk/parca-operator). Nowadays, the preferred tool to do functional testing is Scenario. -> See more: [Scenario](https://github.com/canonical/ops-scenario), {ref}`Write a Scenario test for a charm ` +> See more: [Scenario](https://github.com/canonical/ops-scenario), {ref}`Write a Scenario test for a charm ` ## Continuous integration diff --git a/docs/howto/instrument-your-charm-with-tracing-telemetry.md b/docs/howto/instrument-your-charm-with-tracing-telemetry.md index ef8f4d607..04b794c51 100644 --- a/docs/howto/instrument-your-charm-with-tracing-telemetry.md +++ b/docs/howto/instrument-your-charm-with-tracing-telemetry.md @@ -78,7 +78,7 @@ provides: limit: 1 ``` -> See more: {ref}`File `charmcraft.yaml` > `provides` <11012md>` +> See more: [File `charmcraft.yaml` > `provides`]() 7. Instrument your charm code: @@ -91,8 +91,8 @@ from charms.tempo_k8s.v2.tracing import TracingEndpointRequirer class MyCharm(CharmBase): def __init__(self, *args): super().__init__(*args) - # add a provider wrapper for the tracing endpoint - self.tracing = TracingEndpointRequirer(self, protocols={ref}`"otlp_http"]) + # Add a provider wrapper for the tracing endpoint. + self.tracing = TracingEndpointRequirer(self, protocols=["otlp_http"]) @property def tracing_endpoint(self) -> Optional[str]: diff --git a/docs/howto/manage-actions.md b/docs/howto/manage-actions.md index 530878114..46b00d726 100644 --- a/docs/howto/manage-actions.md +++ b/docs/howto/manage-actions.md @@ -80,10 +80,11 @@ More detail below: #### Use action params To make use of action parameters, either ones that the user has explicitly passed, or default values, use the `params` attribute of the event object that is passed to the handler. This is a dictionary of parameter name (string) to parameter value. For example: + ```python def _on_snapshot(self, event: ops.ActionEvent): filename = event.params["filename"] - … + ... ``` > See more: [`ops.ActionEvent.params`](https://ops.readthedocs.io/en/latest/#ops.ActionEvent.params) @@ -91,6 +92,7 @@ def _on_snapshot(self, event: ops.ActionEvent): #### Report that an action has failed To report that an action has failed, in the event handler definition, use the fail() method along with a message explaining the failure to be shown to the person running the action. Note that the `fail()` method doesn’t interrupt code execution, so you will usually want to immediately follow the call to `fail()` with a `return`, rather than continue with the event handler. For example: + ```python def _on_snapshot(self, event: ops.ActionEvent): filename = event.params['filename'] @@ -102,9 +104,9 @@ def _on_snapshot(self, event: ops.ActionEvent): event.fail( f"Failed to run {' '.join(cmd)!r}. Output was:\n{granted.stderr.decode('utf-8')}" ) - … + ... ``` -] + > See more: [`ops.ActionEvent.fail`](https://ops.readthedocs.io/en/latest/#ops.ActionEvent.fail) #### Return the results of an action diff --git a/docs/howto/manage-interfaces.md b/docs/howto/manage-interfaces.md index 1016554ee..1441b35e1 100644 --- a/docs/howto/manage-interfaces.md +++ b/docs/howto/manage-interfaces.md @@ -255,12 +255,12 @@ def test_nothing_happens_if_remote_empty(): t = Tester( State( leader=True, - relations={ref}` + relations={ Relation( endpoint="my-fancy-database", # the name doesn't matter interface="my_fancy_database", ) - ], + }, ) ) # WHEN the database charm receives a relation-joined event @@ -332,7 +332,7 @@ You should see: In particular, pay attention to the `provider` field. If it says `` then there is something wrong with your setup, and the collector isn't able to find your test or identify it as a valid test. -Similarly, you can add tests for requirer in `./interfaces/my_fancy_database/v0/interface_tests/test_requirer.py`. Don't forget to edit the `interface.yaml` file in the "requirers" section to add the name of the charm and the URL. See the "Edit `interface.yaml`" section in the previous how-to guide {ref}`How to register an interface <12690md>` for more detail on editing `interface.yaml`. [Here](https://github.com/IronCore864/charm-relation-interfaces/tree/my-fancy-database/interfaces/my_fancy_database/v0) is an example of tests for requirers added. +Similarly, you can add tests for requirer in `./interfaces/my_fancy_database/v0/interface_tests/test_requirer.py`. Don't forget to edit the `interface.yaml` file in the "requirers" section to add the name of the charm and the URL. See the "Edit `interface.yaml`" section in the previous how-to guide [How to register an interface]() for more detail on editing `interface.yaml`. [Here](https://github.com/IronCore864/charm-relation-interfaces/tree/my-fancy-database/interfaces/my_fancy_database/v0) is an example of tests for requirers added. ### Merge in charm-relation-interfaces @@ -451,7 +451,7 @@ INFO:root:Running tests for role: provider } ``` -For reference, [here](https://github.com/IronCore864/my-fancy-database-operator) is an example of a bare minimum `my-fancy-database-operator` charm to make the test pass. In the charm, application relation data and unit relation data are set according to our definition (see the beginning part of the previous how-to guide {ref}`How to register an interface `). +For reference, [here](https://github.com/IronCore864/my-fancy-database-operator) is an example of a bare minimum `my-fancy-database-operator` charm to make the test pass. In the charm, application relation data and unit relation data are set according to our definition (see the beginning part of the previous how-to guide [How to register an interface](). ### Troubleshooting and debugging the tests diff --git a/docs/howto/manage-leadership-changes.md b/docs/howto/manage-leadership-changes.md index 8017ea40f..634487675 100644 --- a/docs/howto/manage-leadership-changes.md +++ b/docs/howto/manage-leadership-changes.md @@ -29,7 +29,7 @@ To have the leader notify other units about leadership changes, change data in a > See more: [Peer Relations](https://juju.is/docs/juju/relation#heading--peer) -{ref}`note status="Use the peer relation rather than `leader-setting-changed`"] +[note status="Use the peer relation rather than `leader-setting-changed`"] In the past, this was done by observing a `leader-setting-changed` event, which is now deprecated. [/note] diff --git a/docs/howto/manage-storage.md b/docs/howto/manage-storage.md index 19ddc8933..1d39ac19e 100644 --- a/docs/howto/manage-storage.md +++ b/docs/howto/manage-storage.md @@ -127,7 +127,7 @@ has attached the new storage. ## Test the feature -> See first: {ref}`get-started-with-charm-testing>` +> See first: {ref}`get-started-with-charm-testing` You'll want to add three levels of tests: @@ -161,7 +161,7 @@ def test_storage_detaching(harness): # Simulate the harness being detached (.remove_storage() would simulate it being removed # entirely). harness.remove_storage(storage_id) - # Asser that it was handled correctly. + # Assert that it was handled correctly. assert ... ``` diff --git a/docs/howto/manage-the-workload-version.md b/docs/howto/manage-the-workload-version.md index 988f5d109..96613461c 100644 --- a/docs/howto/manage-the-workload-version.md +++ b/docs/howto/manage-the-workload-version.md @@ -107,7 +107,7 @@ def test_workload_version_is_set(): "webserver", exec_mock={("/bin/server", "--version"): scenario.ExecOutput(stdout="1.2\n")}, ) - out = ctx.run('start', scenario.State(containers={ref}`container])) + out = ctx.run('start', scenario.State(containers=[container])) assert out.workload_version == "1.2" ``` diff --git a/docs/howto/run-workloads-with-a-charm-kubernetes.md b/docs/howto/run-workloads-with-a-charm-kubernetes.md index 90d5d33e5..d44af0329 100644 --- a/docs/howto/run-workloads-with-a-charm-kubernetes.md +++ b/docs/howto/run-workloads-with-a-charm-kubernetes.md @@ -5,7 +5,7 @@ The recommended way to create charms for Kubernetes is using the sidecar pattern Pebble is a lightweight, API-driven process supervisor designed for use with charms. If you specify the `containers` field in a charm's `charmcraft.yaml`, Juju will deploy the charm code in a sidecar container, with Pebble running as the workload container's `ENTRYPOINT`. -When the workload container starts up, Juju fires a [`PebbleReadyEvent`](https://ops.readthedocs.io/en/latest/#ops.PebbleReadyEvent), which can be handled using [`Framework.observe`](https://ops.readthedocs.io/en/latest/#ops.Framework.observe) as shown in {ref}`Framework Constructs under "Containers" <4554md>`. This gives the charm author access to `event.workload`, a [`Container`](https://ops.readthedocs.io/en/latest/#ops.Container) instance. +When the workload container starts up, Juju fires a [`PebbleReadyEvent`](https://ops.readthedocs.io/en/latest/#ops.PebbleReadyEvent), which can be handled using [`Framework.observe`](https://ops.readthedocs.io/en/latest/#ops.Framework.observe) as shown in [Framework Constructs under "Containers"](). This gives the charm author access to `event.workload`, a [`Container`](https://ops.readthedocs.io/en/latest/#ops.Container) instance. The `Container` class has methods to modify the Pebble configuration "plan", start and stop services, read and write files, and run commands. These methods use the Pebble API, which communicates from the charm container to the workload container using HTTP over a Unix domain socket. @@ -212,7 +212,7 @@ See the [layer specification](https://github.com/canonical/pebble#layer-specific To add a configuration layer, call [`Container.add_layer`](https://ops.readthedocs.io/en/latest/#ops.Container.add_layer) with a label for the layer, and the layer's contents as a YAML string, Python dict, or [`pebble.Layer`](https://ops.readthedocs.io/en/latest/#ops.pebble.Layer) object. -You can see an example of `add_layer` under the ["Replan" heading](#heading--replan). The `combine=True` argument tells Pebble to combine the named layer into an existing layer of that name (or add a layer if none by that name exists). Using `combine=True` is common when dynamically adding layers. +You can see an example of `add_layer` under the ["Replan" heading](#replan). The `combine=True` argument tells Pebble to combine the named layer into an existing layer of that name (or add a layer if none by that name exists). Using `combine=True` is common when dynamically adding layers. Because `combine=True` combines the layer with an existing layer of the same name, it's normally used with `override: replace` in the YAML service configuration. This means replacing the entire service configuration with the fields in the new layer. @@ -247,7 +247,7 @@ In the context of Juju sidecar charms, Pebble is run with the `--hold` argument, ### Replan -After adding a configuration layer to the plan (details below), you need to call `replan` to make any changes to `services` take effect. When you execute replan, Pebble will automatically restart any services that have changed, respecting dependency order. If the services are already running, it will stop them first using the normal [stop sequence](#heading--start-and-stop). +After adding a configuration layer to the plan (details below), you need to call `replan` to make any changes to `services` take effect. When you execute replan, Pebble will automatically restart any services that have changed, respecting dependency order. If the services are already running, it will stop them first using the normal [stop sequence](#start-and-stop). The reason for replan is so that you as a user have control over when the (potentially high-impact) action of stopping and restarting your services takes place. @@ -822,7 +822,7 @@ Caution: it's easy to get threading wrong and cause deadlocks, so it's best to u To send a signal to the running process, use [`ExecProcess.send_signal`](https://ops.readthedocs.io/en/latest/#ops.pebble.ExecProcess.send_signal) with a signal number or name. For example, the following will terminate the "sleep 10" process after one second: ```python -process = container.exec({ref}`'sleep', '10']) +process = container.exec(['sleep', '10']) time.sleep(1) process.send_signal(signal.SIGTERM) process.wait() @@ -832,7 +832,7 @@ Note that because sleep will exit via a signal, `wait()` will raise an `ExecErro ``` Traceback (most recent call last): - .. + ... ops.pebble.ExecError: non-zero exit code 143 executing ['sleep', '10'] ``` diff --git a/docs/howto/write-integration-tests-for-a-charm.md b/docs/howto/write-integration-tests-for-a-charm.md index c4d76d7d1..1eff1607a 100644 --- a/docs/howto/write-integration-tests-for-a-charm.md +++ b/docs/howto/write-integration-tests-for-a-charm.md @@ -7,7 +7,7 @@ This document shows how to write integration tests for a charm. ```{important} -Integration testing is only one part of a comprehensive testing strategy. See {ref}`How to test a charm ` for unit testing and {ref}`How to write a functional test ` for functional tests. +Integration testing is only one part of a comprehensive testing strategy. See {ref}`How to test a charm ` for unit testing and {ref}`How to write a functional test ` for functional tests. ``` @@ -66,7 +66,8 @@ Use `pytest` custom markers to toggle which types of tests are being run so you

Test build and deploy

 To build and deploy the current charm, in your integration test file, add the function below: -``` + +```python @pytest.mark.skip_if_deployed @pytest.mark.abort_on_fail async def test_build_and_deploy(ops_test: OpsTest): @@ -78,7 +79,7 @@ async def test_build_and_deploy(ops_test: OpsTest): Tests run sequentially in the order they are written in the file. It can be useful to put tests that build and deploy applications in the top of the file as the applications can be used by other tests. For that reason, adding extra checks or `asserts` in this test is not recommended. -The decorator `@pytest.mark.abort_on_fail` abort all next tests if something goes wrong. With the decorator `@pytest.mark.skip_if_deployed` you can skip that test if a `--model` is passed as a command line parameter (see [Run your tests](#heading--run-tests) for more information). +The decorator `@pytest.mark.abort_on_fail` abort all next tests if something goes wrong. With the decorator `@pytest.mark.skip_if_deployed` you can skip that test if a `--model` is passed as a command line parameter (see [Run your tests](#run-tests) for more information). `ops_test.build_charm` builds the charm with charmcraft. `ops_test.model` is an instance of `python-libjuju` 's [Model](https://pythonlibjuju.readthedocs.io/en/latest/api/juju.model.html#juju.model.Model) class that reference the active model tracked by `pytest-operator` for the current module. @@ -97,11 +98,11 @@ As an alternative to `wait_for_idle`, you can explicitly block until the applica

Deploy your charm with resources

 -> See also: {ref}`Resource <12734md>` +> See also: [Resource (Charm)](https://juju.is/docs/juju/charm-resource) A charm can require `file` or `oci-image` `resources` to work, that can be provided to `ops_test.model.deploy`. In Charmhub, resources have revision numbers. For file resources already stored in Charmhub, you can use `ops_test.download_resources`: -``` +```python async def test_build_and_deploy(ops_test: OpsTest): charm = await ops_test.build_charm(".") arch_resources = ops_test.arch_specific_resources(charm) @@ -151,7 +152,7 @@ async def test_my_integration(ops_test: OpsTest):

Test a configuration

 -> See also: {ref}`Configuration <12734md>` +> See also: [Configuration]() You can set a configuration option in your application and check its results. @@ -172,7 +173,7 @@ async def test_config_changed(ops_test: OpsTest):

Test an action

 -> See also: {ref}`Action <12734md>` +> See also: [Action]() You can execute an action on a unit and get its results. @@ -210,8 +211,8 @@ How you can connect to a private or public address is dependent on your configur > Example implementations: [mongodb-k8s-operator](https://github.com/canonical/mongodb-k8s-operator/blob/8b9ebbee3f225ca98175c25781f1936dc4a62a7d/tests/integration/metrics_tests/test_metrics.py#L33), [tempo-k8s-operator](https://github.com/canonical/tempo-k8s-operator/blob/78a1143d99af99a1a56fe9ff82b1a3563e4fd2f7/tests/integration/test_integration.py#L69), [synapse](https://github.com/canonical/synapse-operator/blob/eb44f4959a00040f08b98470f8b17cae4cc616da/tests/integration/conftest.py#L170) > See more: -> - {ref}`Charm development best practices > Fetching network information <12734md>` -> - {ref}``juju` CLI commands > juju expose <12734md>` +> - [Charm development best practices > Fetching network information]() +> - [`juju` CLI commands > juju expose]()

Run a subprocess command within Juju context

 @@ -274,11 +275,11 @@ Using the new alias, you can switch context to the new created model, similar to > Example implementations: [`charm-kubernetes-autoscaler`](https://github.com/charmed-kubernetes/charm-kubernetes-autoscaler/blob/8f4ddf5d66802ade73ed3aab2bb8d09fd9e4d63a/tests/integration/test_kubernetes_autoscaler.py#L31) > See more: -> - {ref}`Juju offers <12734md>` -> - {ref}`How to manage clouds <12734md>` -> - [pytest-operator | track_model](https://github.com/charmed-kubernetes/pytest-operator/blob/ab50fc20320d3ea3d8a37495f92a004531a4023f/pytest_operator/plugin.py#L720) -> - [pytest-operator | model_context](https://github.com/charmed-kubernetes/pytest-operator/blob/ab50fc20320d3ea3d8a37495f92a004531a4023f/pytest_operator/plugin.py#L480) -> - [pytest-operator | forget_model](https://github.com/charmed-kubernetes/pytest-operator/blob/ab50fc20320d3ea3d8a37495f92a004531a4023f/pytest_operator/plugin.py#L812) +> - [Juju offers]() +> - [How to manage clouds]() +> - [pytest-operator | track_model](https://github.com/charmed-kubernetes/pytest-operator/blob/ab50fc20320d3ea3d8a37495f92a004531a4023f/pytest_operator/plugin.py#L720) +> - [pytest-operator | model_context](https://github.com/charmed-kubernetes/pytest-operator/blob/ab50fc20320d3ea3d8a37495f92a004531a4023f/pytest_operator/plugin.py#L480) +> - [pytest-operator | forget_model](https://github.com/charmed-kubernetes/pytest-operator/blob/ab50fc20320d3ea3d8a37495f92a004531a4023f/pytest_operator/plugin.py#L812) @@ -286,7 +287,7 @@ Using the new alias, you can switch context to the new created model, similar to ```{note} -It is not recommended to use `ops_test.build_bundle` and `ops_test.deploy_bundle` until this [issue](https://github.com/charmed-kubernetes/pytest-operator/issues/98) is closed, as it uses `juju-bundle` which is outdated. You can deploy bundles using `ops_test.model.deploy` or {ref}``ops_test.juju` <12734md>`. +It is not recommended to use `ops_test.build_bundle` and `ops_test.deploy_bundle` until this [issue](https://github.com/charmed-kubernetes/pytest-operator/issues/98) is closed, as it uses `juju-bundle` which is outdated. You can deploy bundles using `ops_test.model.deploy` or [`ops_test.juju`](). ``` @@ -340,8 +341,7 @@ firing rate with `fast_forward`. Inside the new async context you can put any co > Example implementations [`postgresql-k8s-operator`](https://github.com/canonical/postgresql-k8s-operator/blob/69b2c138fa6b974883aa6d3d15a3315189d321d8/tests/integration/ha_tests/test_upgrade.py#L58), [`synapse-operator`](https://github.com/canonical/synapse-operator/blob/05c00bb7666197d04f1c025c36d8339b10b64a1a/tests/integration/test_charm.py#L249) -> See more: -> - {ref}`Event `update-status` ` +> See more: > - [`pytest-operator` | `fast_forward`](https://github.com/charmed-kubernetes/pytest-operator/blob/ab50fc20320d3ea3d8a37495f92a004531a4023f/pytest_operator/plugin.py#L1400) @@ -370,7 +370,7 @@ There are different ways of specifying a subset of tests to run using `pytest`. tox -e integration -- tests/integration/test_charm.py -k "not test_one" ``` -> Example implementations: {ref}``mysql-k8s-operator` <12734md>` +> Example implementations: [`mysql-k8s-operator`]() > See more: > - [`pytest-operator` | `skip_if_deployed`](https://github.com/charmed-kubernetes/pytest-operator/blob/ab50fc20320d3ea3d8a37495f92a004531a4023f/pytest_operator/plugin.py#L139) diff --git a/docs/howto/write-scenario-tests-for-a-charm.md b/docs/howto/write-scenario-tests-for-a-charm.md index 4cbcf7e89..17d3c697e 100644 --- a/docs/howto/write-scenario-tests-for-a-charm.md +++ b/docs/howto/write-scenario-tests-for-a-charm.md @@ -38,19 +38,17 @@ def test_charm_runs(): ``` > See more: -> - {ref}`State ` -> - {ref}`Context ` -> - {ref}`Event ` +> - [State](https://ops.readthedocs.io/en/latest/state-transition-testing.html#ops.testing.State) +> - [Context](https://ops.readthedocs.io/en/latest/state-transition-testing.html#ops.testing.Context) ```{note} If you like using unittest, you should rewrite this as a method of some TestCase subclass. - ``` ## Mocking beyond the State -If you wish to use Scenario to test an existing charm type, you will probably need to mock out certain calls that are not covered by the {ref}`State ` data structure. +If you wish to use Scenario to test an existing charm type, you will probably need to mock out certain calls that are not covered by the `State` data structure. In that case, you will have to manually mock, patch or otherwise simulate those calls on top of what Scenario does for you. For example, suppose that the charm we're testing uses the `KubernetesServicePatch`. To update the test above to mock that object, modify the test file to contain: diff --git a/docs/ops.rst b/docs/ops.rst index 8ac0abcee..29d9c6530 100644 --- a/docs/ops.rst +++ b/docs/ops.rst @@ -9,12 +9,12 @@ The library (`available on PyPI`_) provides: - :ref:`ops_module`, the API to respond to Juju events and manage the application; - :ref:`ops_main_entry_point`, used to initialise and run your charm; -- :doc:`ops.pebble `, the Pebble client, a low-level API for Kubernetes containers; +- :doc:`ops.pebble <_ops_pebble>`, the Pebble client, a low-level API for Kubernetes containers; - the APIs for unit testing charms in a simulated environment: - - :doc:`State-transition testing `. This is the + - :doc:`State-transition testing <_ops_testing>`. This is the recommended approach (it was previously known as 'Scenario'). - - :doc:`Harness `. This is a deprecated framework, and has issues, + - :doc:`Harness <_ops_testing_harness>`. This is a deprecated framework, and has issues, particularly with resetting the charm state between Juju events. You can structure your charm however you like, but with the `ops` library, you @@ -30,9 +30,9 @@ integrations with other services, and making the charm easier to test. :maxdepth: 2 self - pebble - state-transition-testing - harness + _ops_pebble + _ops_testing + _ops_testing_harness .. _ops_module: diff --git a/docs/resources/create_a_minimal_kubernetes_charm.jpg b/docs/resources/create_a_minimal_kubernetes_charm.jpg new file mode 100644 index 0000000000000000000000000000000000000000..2b7e7fffa47e0e745f18af0b2a45c8146ab01cd7 GIT binary patch literal 75153 zcmeFaWmH^C*Dku+=tdfMcXxL}aCf)h?he5df+s-&K|>(81h?Q4A-Dy%Bm@YSAb}va zv-h{p`<^@QIOBZpIX~_XW;0gJRaL8Mt~INws~69B=FO*@H2_ytQAH5|fdBx6`T#c@ zw#O=Ra#q^9T8b*_3V#*Yes;5U_XYC;fV+pEkFL^PT4NJaTKE#GJ`R8fPy)Etw!U5( zdW!l0O4Q{v^iT=-kNz(?o&D1~08DZHY3pD4|63Z;&fUWfWgso8c9NaFuPsWiLg{OE z?slkra1u)Q3-t5y0{{q01qgjm2BLJrKf2A|I_n?Z@o!!7uR8j=a;Q4m06>f7_?OOL z{ztd@OP6!{quV<9xC4MKs$GE5*2@Q#+Ge425gXS(ec6uxY3FZyP#OT>>RJc-IobPI zd)eFnmGNH(JU|z)1_A*;zzMJid{DXm#)ICb@{eE;~$(c0bJn)Z*% zs-hq#%qJ!+Ol$A$?;K$5X7Az0_V1l>q3qNLtN>qBss!8xXaPr*t?sCgHA?>{mlaR} z6aYCu7~liMP)A`@nLXeQ_yf*B0IH@N>dXUmum4li|K81iuK(NpA$0Zs+|&PP3;i$K z|FQ9ZWdENlf5!UXt^Ab!YYc;ki~g18nGOJ=`>1jBsVD0|3q4|H`v@1pqBi z0DyGa*54=K@Ah#2Bp}pGfdVi94S)ma00w{wU;)^ud4-FbSp)zfKm-s2qyQN}4p5+G z8Z|(JdfIdV1HcF{0n7j^>M64W8~_)<4e$WG06!oA2m(TY2q22`jW{3yNCDD-3~&eK zO*udw-8 zI-nkC1e$4bTO213f@5&<6|vgTN5*7I+VQ0EU5&z$h>Vi~|$E6fh0U z0CT`6U>;ZmmVjko1y}>tfeqj@@CEn^d;_+DUEn*g2OI#0z!C5hI0jCEGvF6+0sIE8 zfNS6e1OkCUP!J3R2O&ThAWRSvgbl(4;eqf$gdidi35X0t4x$86foMRqAbJo3hzZ0D zVg<2*I6zz=ZV)es4^?u0C|GELB1e=P#`D>6au;r3I|1kqChdA zhoDEGcu*oJ36uhQ0!js?gPwu1K+i!rpgd3ks1Q^HdIc&4m4hll)u0;CYfwF?5!4K7 z1+|0TfVx0Epgzz5XbAKU^a1n{GzuCAO@gLDv!G9)1<(>`6|@f81bqQ*fwn=rpgqt5 z=m_)^bPDV017h7zxG!j9oPZv40Z*( zgT25$V1IBRI0Sqj91e~G$ADwOao|L7GWZEN6`TRi1ZRVD!3E$#a51<9Tn?@T*MMJx z8^F!rR&WQn6Wjyt0}q1Vfj@vpz+>P^@C`d0zLy@ zfUm$e5HJJ=K|nAdNC+;3073*Ig-}4KA#@N12s4BY!U^Gl@IeG2A`o$i6yy%%E<^#M z3{it~Q#2jJ;v4Pk_oFJ|c4~RF!4-yCofrLRKAu*6xNIWD7@)(i|$$(@* zav=GTLP#;B6jA}HhP;L}K$;*P*12YGyoa`4TDBPA3z^L6QL>4RA>hDIW!mg0$Kzu zg;qdopmoqDXe;y$v>VzFeGB~n9feLnXQ1=YCFmM-6S@W6h3-RtKu@6O&?^`KgTfFn zOc)M~07e2Mhf%}mVN5VK7#EBWCI}OSNy6^H3=3tAk zRoG|PH`sUBA?z6T3wDVHph3|PXjo{tXoP5_XjEu)XiR8qXk2J~XhLY>XwqnR(G<~C z(KONY(2UT`(5%qx(45fR(7e$6(Sp&!(4x>{(Gt*7&{EMd(Q?oV(2CK@(5let(3;TN z(7Mq2(1y^4(Z41Qdaez((LBND!0=Is`L<1Hp?B zLWm)x5poD+ga$$vVT3S8SR)(|t_UxLKOzJXfp~z3N2DOq5Lt*k#7jgeq6$%mXhw7( zdJuz%4~Q|u3}OMXiujD!M(iVgB7PyR(IMyvbR;@HIte-zIz2iIIwv|mx(K=?x-7a9 zx;nZJx*@tbx;453x+}UjdH{MTdK7vrdLsH$^k?Wf=!NJd=vC--=*{SF(0kF}qJKo6 zK%YZjLf=5&Lf=FGf&L5q3Il?Hz`({Jz#zk*#$d!?!{EUX#1O~0gQ0+-hM|pNh+&3d zjp2ylhT($|gb{`jgAtFBf{~7qjZuK{3ZoL^HAXYW8;o9zw-_TBQy8BxRxmze>|h*X zoM8OM1YyE4k(l_Hq?pv0jF{}0yqH3m5}2}>N|+j$dYC4dR+#pfu9)7K0hsqOqcP(! zQ!vvpvoT*_mS9$4)?>C}c3}=+e!v{ZoWoqk+{E0*Jit7`{EY>|f@2}E2(ZYpXt9{F zII;M#M6qPB6tL8=bg+!DEV1mcT(G>b0|d)*{vh);FwutP`x?NH7wC#6}V!DUkF?RwNHn2q}TQi&RExAq|k`NE@Ux(i0hg zypN1QCLo_6Gm&}7Vq^uf4%v$ALJlH_k(0=IV^JJ%zo1 zy^j42`vChC`w9n&gNcKWLxw|(!-B(&BZMP~BZs4kql06NV};{@8!IQuv!IG4B(TufYiTryl*Tvl8jTwz=(Tm@Wp zTs>SfTpL_xTyNYU+z8x9xGA_9xOuq6xK+3fxb3)oxbJZ%a6jR$;%?#Y|KGi-3@Tl7NwblR%I_l0cq7oj{+!oWPF2 zjlhrKKEVTmB!UcrJc3sQ)dWohodiP!qXe@AD+FH&_6g1iZV2IoIE18xw1ljLyo6$e zvVvLbRK@*)Z*iY7`RN+Zf4DkiEXY9i_)8X_7a`b4xwv`zGb z=r=Kh7>k&Qn2MO0n1@)D_ztlOu`aP0u`RJ1u|IJ*@gw5L#973J#1+I1#BYcPiARWM ziC2lg5g!p>kU&VVNQg+NNmxjDNyJF*lBkjBlUR^Aka&^=kwlRskff31lDs0RA!#A$ zA$dfh!K$=FHM_NK! zOWH=-M>B3B~UCO0LwBX=hcB#$CbAWtXH zCodzfBkv#|Bp)OHM7}}(o&1ynpg^Y}prE2)q2Qwsr;w-6q%fwip>U%JpopYMph%<0 zqbQ}Qqv)U*q!^=^r`V*}qd21kQDRaOQPNPdQ3_B>Q7TdDP?}RZPOgTfjO1VRMOnFU(pu(r3qGF-qqmrOfq|&A`rLw2;qza~bK$Sw3MO8#q zP1QoxNA;0vmTH}9m+F)npvIskqNbr{rxv7^p;n>Rr?#YaruL-{r;ejeqt2r)qpqj! zq<%*|Nxe+HMg4>NiUv-DPeVz=Lc>oZNuxxgOJhOfNaI5jM)Qa!l_rm-jHaHZlja@G z6wNZtH=3U`*R<%ggtXMOY_x*3GPJ6+2DJBRU1|MkBWV+9pV1c5R?)W5_R)^eexlu^ z-KRaLgVN#9k<&5J@zP1qDbnfCS(N`%JJb8oN75(KKcjz1UrpaiKR`c5zd-+${)qmH0l`4PK*PY! zAj}}kpuu3wV8`Ie5XunCkjjw9P|ncE(8DmyFvqaLu+MPL2xG)!q-11e6l9cPRAV$` zv|;pM3}$@D_>?h^v5c{iv4?S(agK45@qqC+6B-jf6Ezb%lMs_ElLnIslRc9+Qy5bm zQ##WNrYfdZrU9mLrX{9tOvlUsGZr%`Gb1xEvm~=Jvp(}ZW;f;_<_FAAm~)xSm>ZdU zm_IUqV*bK>#C*kq&O*dO$HK)T&Z5Yo%VNpm!V<_5!}6FVhoy|Ak)?-agk_%P3(FD9 zH7f=yF)ITr533}rGOIqTHLE*oFl#JpDr*62C2K3|0P6(nGV2cODI0_hhmDerl}(6E zmQ9n*l+BUNmo1Vlne91S30nhOH`_nWaSj%yvwP@Y0l})8Nm5~^C@ROXC-GV=OE`K=Nji8=LHv>i;#<+ zi-${+ONGmb%Z|&NE1WBd>p52mS0h(1*C^K#*EZKFH{QUfP__g>g_+9yf`5*CT@E7sd z@%QkL@-Ol4@c$Bk3lIq~2=EEW2xtnJ3%Cdb2|NLDXOLfoQsD zk!ZbWujshws_4Gxl^9ZtQj9}PTuepGSjFt%R>cv_z^zp+ucT zkHomds>FfBjU=`twIsKsl%$5Fg`}J0eaR%r9LY+_4#{E31<4)B3n>gKGATAGF)0-( z6DcRDAgMU1EU7Z7R;hPVb5dJUXVP$K5@{A`5osl9BWXwJK-A zgp#_FrIM#oq|#HRBBe&9L8UpRZ%XINn97vO+{!Y_+R8S{zRC}kpDC9qwBfJ(5nclD5w~zIH`oFB&y`8)T;ETOsZ_EoT$Q8NmbcZB~>+4tyH~LV^q^sOH^A` zKd3IM?y26W;i=K9392co8LPRd-B(LiD^ROb>sOmm+fqAM$5f|M=TVnc*HgDw4^)p= z&rz>d?^d5s-&8-*fNPLxaA-(rXlYn$_-Z`V$kM3Lc%w0@v8M4u6Q)U`$)+ixsi}ES z(?|1xW~OGjW{2j8=9=aYEvOcW7Mqr&mZsJ{EnlsNTA5lET5q(*wAQtbwb8W6v^lh; zwY9Zvwf(i@w6nFVwY#+^wZCYe>0szk>hS2?)zQ~+)Cti^(kalX*BQ{6)7jCv(#6rG z*A>!L)-~02*NxOo(=E|$)g9Jd(LK_G>XGQN>q+Wq>DlP{>&5Bi=+)@;>P_oy>0Rg} z^=b75^cD3@^xgC$^i%a;>9^^B)L+&AVE{8AHQ+FiHqbS&HwZFFG{`roHyAYdWUy;+ zV~B6aWGHH=ZfIrbV;F1r+_1{9$8gGU%kaVoX+&ovXrydpX5?WMZIoeDZuG`z+-TG2 z%oxL%+L+&1!Pvyu%{bCH&A8OK!+6wq!}!z$-Gs`7&qUtD*u>2w!X(Y4)TF~?)MUft z)D**%%9PJk!PLam-89NH-L%Z~jp?}QXVYJ1SZ1_lf@aEQW@esdF=m-&m1f;$Q)XLc zzs+&Y8O=q_)y?mj`(ibnpnD9Mq56!thDU5oU;68d1-}b#bhOBrDb=#n)sK5< z_bBf1-jly)a?kx<^u1^ID)05&o4&Vw@7fyQn#EeuTHD&*I>h?1b&++e^@#O`^{EY} z4XurkjjD~6jjv6dO`c7?&0Cu#nD22q=k(nf>`dy+yyqF+`PK8vi@=NB>yDR^m%G;kuWYY6uXkQ6UMJp|-t^vL-df%c z-uJ!JyeqtWyyv|4ePBKmKKwo^K2|;fKFL1CJ{>+2KHq$9e2IOzeC2&jeZ76-d<%S= zeMfvh`~LRB_ha+B<7edO;rGxl*RR3vgWtN}FMk|=W`8MveScT~X#eN_ul?Woulk<` zAOjc!Bm(pTTmqs3vI1TQybV|hI0-}sG6qTn>IJ$4Mg=|(d>!~Ua5eBW2s?-=NHR!2 z$TcV?C_AV==zY+7(63f?XWXy8KX`xn{z(`z zj44bi%plA?>|t15SX0@RIPZ@Y(SF2($>A2$2Y_2*-$s zh^&a$5$___BF-c6BH1GEMw&+YL?%WSMRr6^M}CijMNvfwM`=bmL`6hpM!k-D7quRB z9*rN(9xWGb7VQ_E9Q`V~D|$BiAO;ab8zUB@8{-la6O$X$7&8*{HRk#O$phX8Di3TP zggi)lQ1xK&!ODZvhd2*eAId&7edzlz>EWw~oeyUp9>gMI>0-rW^`Pos zJW0YyVokc6WR~Qg^f;+3sV`|M=`vRA3P*}Uie*Yr zN@_|~%23K$%Ee>C$J~#VAKN?*eVp;Q_VN42n~$%akUZgkqW;9;N#v94Cyh@=pL}}? zeoFaNYbXHT9Vq6x{!L3hLgsYCZA@R7Mzxr zR-N`PZ6oa}og|$q3>XY9`u zp51#E@+{-o>u1ByzGMQK6q&-A+L^AIv6+RLZ!%{ykFv0`ShDVBS!4xerDfG*z02Cn zx_(afT=2QpbC>52pTBtC@qG6AQ8qH0HCry*GCMdsJ-ashL-v;(P!45|NRDofdro{# zaZY#6Le6O}UM^>@Qm$=oc<%Gurrhz|-8^_6eV%llNuFQclf25jw|N_R*ZE}mLiyVH zZuyV$i}Jhj7xGUE@CrB!lnd+%A_{T}S_&o$_FrJUV0t0@!s12Hi}V+-UwnMARR}4h zDU>KQD)cE#DXb_QD%>c%eo6LH=%vm}_m}Z6U%l*ox%~3Hh^UCKNTbNP=wVS|QD@P7 z(Md60F;}rlu|simaei@o@oe#rSJ8RW>(a2&?9%4a$!*Z;0wsM7XoAQYA zobuN4>GGor>sEWLb_KLZR<4W90u1eKP$I1tlFDg4L7b?%H2&?$2G^<>y z9#s`r^;WG^{jMgh7OK{*_N-2-F0US{-mC#?sA|M(jB5OAQfq2!KGtm4qSZ3g%GO%e zhSp})Hq}nm9==AtW`C{x+WvL)>ld#(UoX7=RYz3EU#DH?UYA%`RySC;Sr61x*Gtr! z)Cbh3*VomL)qiimXkckjXs~UFY{+luXqay}Yb0#sZ`5vdZ%k|~Zyaj;+yri-ZIWs- zYYJ}4Y-((pY&vYlZsu%OZFXvY*j&`y+q~L*)k4uC)?(D+-;&l+*D}_!*NWN7)~eKM z-x||e*xKE?-1@tXyiK&tu+6V6we5A=SleDZW;*R~-}`;vL2v z0Ua3~4IPsmhi|aoaJ^A`sIb|=ziE;+}+>3-UIZ|^ho!Z_k{Lj_q6qV>N)Er?iK9S>-Fh< z+WWe9tarZ;*~i(Z*5}d}-&fZ6w(n~{te>%8uHU9Vs=u(mr+>BodVp#`a=?5bbRc`6 zZQ#?uuR)SQ;X#8z|H1UZhQX=9A47OUd_&qpo<$SC5*7a?|+lsgE z-)_G{ykmW*^v>~J?7Nb8L+`%4hrMTfFaO@|ea!o!_xaf4KX>`a{%*!VkS4 z)`o#$+TlCH_l6^e3x>OgSBGyt(tMQpX!SATWB$jkk1HRqN2o`nM=VFeNAgFyMpj2| zMrlT6My*C8Mhix}M_0#yG1{>^WB0})$6k!}j;)V_#_7iIj@yh!j~9*ik8e&uCm1K> zC+sI4PQ02Jn)o^ipJbU-o^+awpDdsJFu5~@ImI!hKIJx*JXJk4I(0COGtE1#J?%Z6 zI$b|KHGMooI3qM;I1@ONInz2bKXWllF)J}^J{vZhKifUKHV4en&E1`|oqI6%YHnz5 z>l5M=+b7jeuAh=VReu`&bTE%Q&o{3-?>nD9-!wlr|7(G4L43h%;r>GYLifVjB509e zQGU^WF?O+R@%`e?64nyelID`v($l5-rRk-UWs+slWs~L5<=o}2<<%9?3jK=wiv7x? zm9mu&E8kaGDN*C*D0 zZV+vVY?y3>ZscxsZLDpAHyJk-H=Q;UHmf#AHxEDKe-`{~_&Mlv_UAXBSHFP1Fnm$? z;`k;0OXZi*FNa_8zY2ae{2Kf<=WFNJ)h+NAigLDAA5v*B73HLVS6w3`uD!o$xH;Xqnw^)E%EWj-m;1&yTiv_sF0^DK& zZm|HjSb$qBz%3Tw77K8T1-Qim++qQ4u>iMNfLkoUEf(Mw3vi1CxWxk8VgYWk0Jm6x zTP(mW7T^{OaEk@F#RA-70dBDXx4Zziya2bn0JppVx4Zziya2bn0JppVx4Zziya2bn z0JppVx4Zziya2bn0JppVx4Zziya2bn0JppVx4Zziya2bn0JppVx4Zziya2bn0JppV zx4Zziya2bn0RKPc1^6rO1S&efU*Rl@<^VuH6ct%x1OT3A0|2uFD!zcZpMB7uxDqmv))zXhP6<@)~!_kgPZub2dXuKrtY7E#dwAYkyHKa@aGe<(6<7!(S_ zfWy%cSQuDXm>8ItNNjvuBsLy4CMGU1E*=3P5fKp<4hbnSAt^o~5#gUkKoHb5P#8K4 zhE9mYL=yhD<)#O~Lx4)a^$-vq0LBAB@IW^M02L}s0*a0QPn42>1O|el!4VKJI;z+S z7XU#ZU?>;{!9vG_p*lxJRY2VW9vVKJfQ}DANJP&st3!O(nuI|>MAy*UKk8}5B%`37 zjbD00;}kk6laR2!flqX5+OxvC`iW^4IeFgzBZrqoy$ZH=heu@0tTDx}`uf*VJyD`e zKs}>>vjaeb!w{(32BS(<@K9!=I)-Av&=IIgDDV9_{bL6mAAzjSQ$itYZ=a~by50#m zy{=w85itqBpp9?pO9p|vw$YR8hjM*KOpLT)n11P>Lv;Y#n$ACXl*>BD&O zVkf#xIxuOo_N%Nul<2R5|7!thw4(A50+d9X!nDh;?5lf!Or|t2dfA|>4 znr}6YCD%Y$?m2t)H>ZLS!cqrGfox4I$nOMZP7Z~$II`?}YULa~4P zK%QtRuBEhL@${{deyNOr`w?~Zdem?4KE|Y2F!Dt;F`e_8XvF^l4Db_>M|#gO%$&zF`S7UAE0=wj4~h`Hx@Xqg{UEPpgV*_YErP}`YD_N{vbSw# zFH-X!mp`8rtE=CZFzzweB%h5$jZRX8hz}|z!XH7Hj@UcAmNqX)?3Z*R!~@m(gpw@7>pIaTV&lDtq~f zx{02Rut~AfGTBxUoX6wQZ6K&ScS)0xRps~eGjiwsW88inBL&K+$zICA4Hl`+1)2|O zWCyP=aaBA>M?>ywE4sKVC>0-nnLV@IN}zCOY-&Oi_V7v(Ch0()<23kFMQpn8v6S5n07kEsPBq`^(q;IwR><$mgMYXhK%gRr%UvE+cWYrru z@9~s;C9V+Zg~`z+{j5;4{<@~u+pg5wf#gPdGvYp(Tp`>)U7_$wd$%iog+5?G^{Q~H zVTJRFp246bcM2*3#AaFY-qYMqE2b3aA$**b-Z5R8}+?u5?5djQi1gz zwX~uHo?)RcDA=GsJwhcV$;q$6S%b7CM^lTJ0swp!*^f)s?K!3KdrZjK*ZRxlzdXw; zjTg}6N|IBD?<@o>PCcPK#hnWs)|8+f5+uPt$jp+&drw_fQ}bBI)J018bWU8>CtCut zas%)OQoAlnsHI#!IZX|_7ejeCJgA`&l-4&@yHK-h_9J1gilItqW>V7Jg|6bkR6M^o z!#oAkeD0I+OTQ1;QHO8C+jpplD!;axS8)6D?pmm*K${xcZ7Fpu*vYW&r7GK!&o)(4 zp@MV(P*SVNoSB5>b%#O9rq>fI)oBKD6^jO5_S^{JDJj#vhQ! zSnMKI_bI$XlejiCF6L{F!>enW7W|f_`ea?=8)xn(nEB0~cG$~R6opdSdGRnih&?PU zggz!FE33Vz987r?W4S8$1h{l*D+Qkc6x}Ke54+ZB{g!sMHg!> zgk3p7O(EJeWM)}0Lqg?DweZ_DKI6&1VbOvmyJPY(PKu;RWs4{%(su@0`hcgBxv1;#t?A6% zQl(#WVUvbIIk$+r%VHG?%dn<>imEDz!c6|#4#JU}6HV!vs93)bvUW}Pt%=u~W$IjJ zm|let)eWGxKv&Tk;&ZNJrpd$;nphaCL@c*Tnj~mlO}d>*MgJeK9S_!M`yNrW`YDE6 zIY&$h-T*bx-G=*AS2w_aokpFwug|USXI=?tTzsSXe-(MGudbe=inRWhMXW)~{V(|a zPctD$BO<#mhQC$bYn|vdQj_!-Y^bXr6y(9NOY#~vpTmBDD+|4^9r}XykF>O!ZF@`; z2^vg~&+6z8eRC`+OH!e%YsLL4R$7!TOB*vD9fQAq0Q18^FR;8KDdIFWIN8Up2+Ln3 z5-90qDjbZ&Cs*kY#mWVxlXedGTRy3uSr#I}5gbL$4!QE-@)NE1tuDo-p`%RUL+$ix zpX{9J_TC4x1ZvA|DgQDX53qzgcaf}qpEh0Sx4-8bCZ$4o`5|?y?F3Ye`?hUz`1>gi z*C?)=6jkBz8IItIRHE9ay^D|$t=2(Vw-S3v6w$XgmJ@_ARpl%#HHK)0nli|!y2+Y2Q;4bMmSE-+a)5gjAeE)_Su4OfAV1A9n{%p!InH5ic30A>Niiya zZmZ}5*#f}E&8JG|O<#CvS^ zjix)*jfNHCYGL{fBT|gKLUpR=D_bA8L`c0u^UoxthEqacM2}07j+=9&e4$R6Xy)P` zPK&8Db81mgXXop13Li@D?Z}JGkMZsyVqJtkriq$ zBL(ADTZI8?_GMS6>OiGurkt~d3E8jFPewrj2xz2mB(GX~Wc0!4FY6ls>RNl8a-9Jg z4YK(^Ie7}+?X98o)>5>pSz>Hkw(Xo}D=;u9<%!`R1E{RXIPuqHbM%lpcP=?WHf>zhO<%Uui)uLniEDeaQ>^!)tYb7|(X;fHiNx!OU= z@Wox4SuM=|t1nVVKYh<7D9(r?I4>VpD5n12ZU}$!=p?Q0@CFFhx_^J{{ioH|%5%rh zsAe-)H8PJM$W-N%Jo^11bn|Pt=FruM)vyf1&eP(q3%Au`x+}ZGH!>@Ft26>X0z~#X zu1(Hq^ply+rtGiiaCTW9u3!Ji|8-uFR@d!q^keLOu*=M;#GXoNvt{)e*~0QycGa>k zK^}MBBT;b-{4ef)Zwa57Uw+y8{D->Q%TofpN6EX*J0+5JeALr@$C5Aq`(xpsD{wM@ zvmDZ1ZXbzmtdMw7V)v`ti7e+^@FRA+AM7dV*`%Bo9uVi#FJMCQe+%|z;B^4cvU0t= z=SudL#T#j-NI3j0ve$)RL#Z!Ut^}z z8ykU`^JwfJYX=L7a@P!n*B%38(f=5N4BViEd&!xBLDUI3g_oGf+@P+rA~xZVqt8c4 zrb<^X*ZuH3WK17-FTs@dhu-8^bsD+k#&!EVzU;|XvPPYxBoGU>O$?EvseBTL4Gi*; zfAchu6!KHdJY0Qwe45m}gUz~Ql{VIn*8VJkjQ-IxN)cd7tcE*a5*+}R*0$R7tbbq; zB@(WFtnoPTNx>_vtz9aw>#)kUsmoaRk)UEUn={GEF7e9~TIVZv9k%4}3)<=_IZUlz zHj*_S9?fY7JMF5Gb=E)v-afmi8P@#TdJh*zHl;OblO?IS!nH+WcIS(vM3?T<@3WJm zff@m|dTk-AqMOA-?XKiU$t({BcDKII+ARE(-|4;kQ*rJ`vT~{9iyNSgXJ~m<(dRW4 zky#$|*3912qY&Gv2@-LtU#nXsbv60!{+?q{ow{7J_Gfj~(lMEprtqsv46FJ&yyRM~ z2I*ghR?7bBUjmv6E}#B#3B+$&`1R*&=RcJ5i24XH&S(Ka%3?n@q#!nP>;~X|4sFg@ zXU<+z(BLkdT{0AQZP>(r>oDODJMk5al(Rhj+*T^%MIOElRf=0BIF@v7>UC(|4$zs< zQ=ez(Fi*5HUt9H+3sI!{+3D!`P4)#YN<6J#1)j>?ghmlzfvBZci=i8 z{t=p{5JJGqPqUns#=T)tj0BII^yf}(~U7NHc;b}gl`nr z(B75Qxd}yZ6JFY}ou4?hSGZZ;?A7wxBqnC6d4)w<@)_4 zLPq?JMp#uV@&+i8L48;8W8b%>Af-!o6pCZxBl?qLFnUq zvymz?cVtCQBxbrp0&jo`A)2!2l=agpTeZAbJR`2Mj5!8(0PL+#vH*gCp-fhvVy@SC zjLMp9$epNDYt06`be)AF5AD%}a76iOTN?2Nxs3F%ga*&2Poo~!j-RdrzK2SA#W?I< z)2X~zyle8xpSnayQ8rmd@k5H{a<|>m>E3cj>HW3SKAbR~>d({dhA)rzM8X=nxIMT6 zRUH&en0tR3Q_C*@4D;x|i=d(!yy&zDD``>b?{qBPd|u;S`({FKP>YzN1Dh*v^!<0< zsTp{4!$x&@YFGQRR{x$^*`}L%=9tO$c7di^!w!V;yF{$`@!10cBMn;v&j~yiMC0I2 zd--8M&x`VToP6hCCr-La#F>CM_sP*tE`Icw?C$T_hVrU}76lh7_Jshxk!PnL0-Aj` zWL?GdlzsTpas#Qx?^=kqMYOC|xE~%i7#ZpH8+=*xikhZJH|h2i>!^S<+s^Kcd+&e1 zdTgWy059Ye_$tE?I`5HBKh76dv~-8?#u8*tTHMd5_h=nw zo!_?IZJkYW?~#pUx3Cw@_&K7SD8-bViu6Xl}h?x#g}HN*vNKiYRK>$${w?NR)bTS$=2CIsL?hTAebj%v$VVP=uqu+vHhAc z^E`;;v3-Pl?L@(Sei>y;j;2fHOCgCU(~+=0OYnav_q*q1qTpLb@wH8vpzhrU8Y_1pB(|yM(|6+h6ezA-9BtCQ7Yj0T4~KBNac+QS-3WQq zy2JCUn}Kqw3V{u^Z(g_)LHC7r>eV2Mq`GY&)jTvcG#!dsG8`8tc?OAA_ieC${`;OH zoImOT7HIztE%iEtif8B2U-6Pcb2*QQ1ErKV%YK0-gWMm zedu0h+!4+8rFQ2=RF!8!Xncz>+#Kyik>E%L<6Zj!3dpg2)(?9hQHvngm{Q5< zmALD$1%rmATjtBlnGzn?wu!l#+~e75zq#0UAEN=LMZ-_Odgc75zc(n9u{xKtZxE+( zrgzw896a`V0w-vzdH2f}JyXbRM>RIjOU4>dffx7tv5u>93yb&}VA$ofkcnx|E?={P$@L zMM8(ia^-0Yy?@pwk!xvArZ9^w|C?@P_%#E1{I*Nn@|8)c^7FZrH(vuoTIKN8dE%Cp z`aVnxlSIjF&Ki$D1LT8?w(0 zV*KrH5rH>+`394%zPowu+gKOgYB3Md!Zl+!d0tPoLs#5Cdx`&kDsA<>peN>M<-pv+ zaOCR6y$BZdn1UyN&6&UUE~5YG-etoBELZlJsMbPl$#r#S!BO~@zfg)Npx^4QZ& zMnPY=b*h)tPSyLDs0C!okUVeaCCh`u6jN7$-IHdB3wy7kk80fL&R z)HN%(sk8ViT$T*wHRSEFm`tGuf<#Sxy&}SBZ@cI|+Ftu2ti@>YrYWsMy!8TByfr>* z@TrgJe>_?*#TLKF=j`G=3vlT$#`xN9llYW;D6J+H0__waYW#Qu|4{$DFqC&Whrnsh@>cw2>qAH@cG zU5txT1-!w_r9{iPc;dSHCQRvqJ!0n0-@3~Gt}grIn#zTOn5BEWvZ*3@rHHw<=oy;0 z=6c~MgaybAgDD2YP((l;mVBkIo?n@=q-@NeKCrF-$n5^OW=J@B$0j~2*+F!dDqA2Q zp|ivql;@bJX%0)-G#e@Q_bqf+;>Pbg_j>!8pjqH?Db?La_kQM%k;K0o(h@pbb&Md$muGK(9|bUOUb*av~yyj z--uG3nNB0}t+0w>k0Oq?lKI)O<%svCOaF$A^U})2gu0ekvY2;4On>bBQVDa8@e<+S zXj;)lhLM5zHbLi*jZMCk$M84_d(P~`sTPd+;F*ko1(*0&3lv>S19PHleq@ZtR7CL) zYARbp139(?2gYg$ORQazr1?*Byk7~<=qqJ2+Ys!|9{$&a7CML%mbOISRpd)~`gM*M zX{#15dIp-VQuHJ+1RpESr>8mWB={zA&WI~sNuQy6?V9db67NLauh&s`5z{JtMxM=P zK{(MGZMp_XEleV4qh$~V)NrDN;ab00=gki`Hhx)d4xUlNKi}}g*mx*r?-CH{RcX;Y z_;$2GT{(SUYx+cFahY(mB3rnRK03j0kWgcERuETH%8|M_RmF<*yCMs3r%0{dP>@&r zB;9=a+G{akV%}EQr=?`ivx`igZwYMee2;Uy&>Gkx5X^{`P@_%R#p3+ZFEzR-nWi|- z3-VP9+}v2x8pz1$-iK?VM<3^3uTP}N%&s;~E)BJ2 zG!wcTq$YzD3jVxualcxtNO@V1NTQ^np@XTBs!heob8k!<`#CLJ%J77N=)%{stc!-3 zFAwdHqXL$L?aN-4J>O>ER|H9`xamJ8pTQas)c z%Wgv1R{c<18bujCtG5y`U^ zam-4i-Hq4%+xpez!SzGP4#|ed4d5o&op%F_zvvM~y?teOR)f`q)9!xr7fN}#g%ohM z_06_@+PiB0KKX@pv~2F>7wm<&_&&4vFWs0N_ez{U^2}(7HW}szWo7J168cjr*6Yt# z6B6R-y0~_xMu$l;qJoj>^{chhZhUQf?B$vpT*}lta37;rLC_%KWryJgeNM&{fnxf{ z0lj|9S}Rj4i$Le${fW3vgV0mAr3!9WeFXy}JaK0}J_7MDOP!G#>kWlBUh~aEiyTry z-{8()4T7Hq%PQ^^2)udQ5Fzn3)XaPCn=#D?LPdh2rTnx&UrTb215ek*S|cg($s6EZ z%NBp)PoMn6;=~4W&C^T0j*|B6j=9B|FX54gEwxyU@xSK3-2hK*T3*xCn;tu#$(adw zl5R+e-vEU55BF^z{JriHGSI<<3Z~J|D*0pQF7Ml=@&R&|fZ)>ZT z8^Oye_O{oGY&R%FtE&3!`O0n@w^m0lWh%ajmhP5B`qJg66~qN3J4Nu3ru&7~mJ&0n zH9usx)|Tw?fBtj#pDSZPnQj6Ie0_d%N#3p(5lHdqjI+T0f%bQ+ z{^if=UyU^A**CoFr%aXCGB*E;-}mh_m6(~|Z2Dk-!*TcH$1Bs|kIn*AxuRamt_$~> z-Zc^3bG{y>&$>7{ip#TM*^}2Ioik*(vLQ6@j4?H6>IjJBnm8uC0my731l@mZm!4bG zT`}6w`yaYQWLxmPsw&cO&e14rAdg+#rYj~8>rfkTLoO+Gh>jBO54JYA{3BU;)|$yGV~#oAcf4apQ>6E(B}#5f!tV6b;Pts7)c!6BH|E; zJ!QlN==xP}L4)5yArILdABEwn(u83Oo_a& zAKTwiaEI)L%f*}zy_B<9r1GrM;(q@etg7-cLouOwV3yNa~e4pY2K>QkPCd{_@}%Q>&6vQ+Y~0eagO1mzo`Zcc{yb zZGvv&PNjiZm}iMj$%F{|#+}#ZkGt1<$R%^yJ1w)2){gw@JRY7X0v&jyo=7Sa3g>Pu4qV?SM>QL*-At;z#s~8 z8gy(PEEH+%FBG1;`tfHXNO-?gZ7S-mA6ID`F{r39FhC+ubEYP{vB=OlyCX56D@pQw zoTh)}KJb;Jp8uzawB*4Iwp`;|KSz!Ip7UrzN zt0f19)lNQ^(%n^gW)YR{lkg_(HbJTYaqIQBvqOSI!G-@N%<)9iLFTs1^lZRyKxugp zGK#$MKRndGdk=BH_9jg#tmhU`O{k($Y?2fZR04}c7~sHL=DlCUo_8g~Zg#if zLYOPvDAXN*#zjIuYq1*oL8F<*BV%_ksfw#?0GaTAnrlulq{TsyOSJQ`g}CjJ?*LLj;sc8wxr+U`-mq(x$vST}yP*ZOkcWuD z>BsK1W!u#Rfys&iwizSE2EwG0$>|_D>8?KP2!msJvRKIk4iaQN@oKVB%=le^9m#ym zFHc;s!f;@%0|`59((Oxw9NFGT&tpdH=Mor4k_tn#atG=tsiB~dH6)nQJe!rZ{iH$5 zuAtJ8$v3C4Ub{(i`u+{i5ho8$HP44PnZ|5N;wzBMJ0s6DZI`~`byfeqQ8xZK)h`gb zln0a796NNiJXy@M>Jx3)6Ar=4L>v{comeqII%`el3Op5*oxnw2_Gv?P??dF2F^yUL3wFzppd>>TM${Av2ksnW(i*r`q+D9e+({?(=0jJb?%3yuAUc_}bwIc8{$#&x z)ZHNF-TijPYz|YPp5sj2r_&g=ddjWWrQ5mlB*%FQXCvojk;<~t9E}-GE3EIWtvY&H;uY&R()f>cz0kEm@8bTAU*he+91y@C($I`{Pk}D zZ0deb=kp8MKxsn4>N=niT3Bd!{oEJ_{XCHQaWcI2$jywOA7zejCL<9q^vnHpSlgR1+|>3;DBik{sNq*L9J z`&Pby4HRDxj(IM;eAxT|G^~cu>Ua#8yqVy41D(d&L zJZGw@d=X?JWNId_p?xIReKx69kbkx8SNS4grbf=52L%1;6rfu6u#v;mj6r?2gGkiJ zV?eB@oos_Yvq2Wui#~1A@7z?o?mXOOs{ERlS73mhi2meMEQYFS;x^=&1mgj@Vw(|G zmMCGu!x(#y+Dm`aaZozpk9 z$$Avf@%kqAL9MZ0PHL$raavkOQO2lijJtFrvB_P-bO;Gqy@MPNN}>;R=f@<2U+BQ{ zSbF+RMYhf72OFfjS%DJaF_S<&w9So=C?1#c1jSp8UZ!)X1zTV71Rr&4C2){&Vtq-|+$b7{l( zRK1y7CHpWCcag0#51g#;#^y=WP$ydLX00+MSXZlFOoc2(-0|V^!qS753J<@`42jTc zeV*4Fl4;8o#`krp+HxZ6>>56k&$wHAB{Sy5FaoZc1-E4n&Y}6_)ms&4ll6*q04R>x zxbU0>Z8aBg^t+TiV}R>w-^$*iuWu8P;LK8?xliAGFALvVBj1a?ea!d$hV$L6 zJs*QEU%S+U1;QA8HKR?Qmm~J)+PtNSBJ>J#%y0Yy1TZQq?ey~7*#u%PJt!E%6O z{^)3sJ8tDiC*pIWsRXF;1HGA?(bjG~_`+BLFIdcB>k95lcUQE6T z{Y$fnL?*%yGPfoTov~pw_2@bLKf)JzncO7)hb?CQB>u}3P01?ordTiig0m>Y%g-t_ zRm|hj-tza4A;A+%i+6#7f8ruxOs{>`}S6M{be;dX4GL5O?s?)c{OAEwB*y!f2N zrNm@r`Cxra>S2RJ_-j*^$5GS0Dd~h{YK`A(a~i}2B;&;u>BI;w)dfeNaGpa8?o6w2 zLnfuIF5~SRzuyAoZP|T1oC`&iA=~`+u5amYP?>thPWTci-ctvY{aD5xQXt@e`NzY< zkgT}(vUl2_mV%}354}dM+vZgESY!u~^Qflxym>eJtqakL5fcSu`H%?e+_)#3-+c{ZUl&&e?V4fuNCY5puY|^=6qCMulOh=-+fh{4jA}W zHq2G=NG68w?zg_f1_zdW_@Y%9S^Sw8qTl6b(~_S%5c@>Pg{(|JRyW(5^sAFm*M7dx zQSF3L&!Ik(s1~Czwb1UM2mBp^E9F8M8C$>#wm*}Vm9buuZmk}+4-l1xmo})`EfsW& z;|}?MuHqm`!+n}vI!WJz2RoJ=GrOvr5-DD&%f0ay8xoMyDg~SO4k&WFOi1L$ulI1V zi4>Zy`?S=feEiZliqn&`oH0zk#9H{wH$oWE%c5v}rPfF^e)c-NKlNy-=QPYGTX4TY zs^6;24O+;>)TcvloR?))Tbs*4Aj<$M=P6={Gi2)i+a;YGTZe~^YR;F^z$MPobm1gy zv7@G6U|UC(Q6j$@`n_29pM#dh1Z#n(1fRN&y1Sac(9~Ps*(Q&HHVTha`IaAEW+;we;(14#4dd*=Y=VA59x@=yeFpiGgb$y2tSZ--1h$X5uv5I|H!haVkwEkig{wN z4shE70swf@$rqPj$;_m*RDuuaaf93XSauIFnP)MlXZo(eTl=i9 z*U?taLMDaS0e>Q55gJi)M;~==bPNHC6?zu279_YTtv2k@qUB#QA8t4Dy+uE5`$#Rb zykkw)(HgG=>bMl`M2Ay4`&uT1=uJjS4gAO$d8V{3vTtU2du2PY3+D~OSQR}Hp}%&G zj3N4)AxFg&;Za$BR7PXe(3#grl}DBiSB}t@mt2^HW>@|Bdk(fnvR@Dq*2A%Q`CMVr zjY1Z1zH?*F$dG~(SmlQLEG1hZ?|m2bW=)k5O8YlFZ(^Gm!dPJZQ^b}36P^F_2ANgF zcCDaF!=!fsEF7s7+-rILG}D3Ca6?g+{YT(CP3v~Te@4Zc*<*o>K{bu%zX8T(zX2Kd z7{LEzYyaCOA)~-8a^viRF<%zNf&2%{IPBVSf=)CTkO^vch^j)gwK08aUT+jgINvu~bjxbH&oIy9%zp*`Gl9W5kB~ zM#zLB8Uy?OZ{|{mqP)3krBW3%I4gCPFcZL`^41Fyw{qdsVNhcxgMKs$z8p>ab_c;TZGKQA1bI3QP?2#G?v?-QJ z{JMANpmAPb&2EfZVCSsyR_{&v>04~zT#qs}KGp^(1)3RfyYmBcJuYNAzy_X2Z)F=& zk~jl7QzFe^J;wGj8Xl#0^7MyItM zT3KB`erdWJ4*;&QZGh)0qg9N{_+1xU982Zp77MFMn<*1u{OPZ(1)N;PnCO)m8lj;# z276V01NsW4rC;fsk;wJahYV`(rC(-mE#p=(K*2gD!KoWU^!tWAUmQU$3(fxe5e~-p zM6A4W1T^lG_vdPH@MK$`m5EFGfk__OjKe-BqY2rBOidE0$F`ChfAI?CB(0N3=J|=@ z;;Y{Y(y?N4Nxg+LOA@kPx2Rn=zL&Hp-zeyr$G#ydX7fSjjwiX-BtO=w{>fA6n(+!_ zA?!;TYXha{{UF4L`A>*#wUJ7@Gpr!Lhc#dULBMFsbSV&Z{^ z?}zJa2H(0&Opn$}RcF4-o|5|}OAALEUGjK4*OqaTM&_kx>#9G8-iI*7xv{>Z-=d25 zgFJckw9d!7t{!-4o9Z&g_@1fKFTo;;hXs7prL2_3Q(#`&tXc|7kScxcd!eht(PD7joJ9o%BXsASHI2_n$q}@ZUrue9@ zIt{9!J@CYh>adfLt9w=7N*xGyf_TZi_HS7M{>+CRveGFYyt)ZPBR-7+w(mocbDX){ zAOMhYXCmvJ+Y(qWed?*$z3n3x5;AOSxOt=1Ju?$V6sGHOgv3q6=XB_6D zdqfdO_{F+)8p(i)u`774>CX_W8PhbZ`}l{rc~vTn3hWpCpv!&l4GIh55+h&MGjM=X ze6_IysaMx*`|i?h=Bnnhfvo_{%&l7tq59lS!(fJ5RczWK(qId(g%{Wz^Luwxn;F6B zxFOdUa9{Jer}JzJH|4hd+`O&dC1!PnVDeT{-NPx(rCO_ZCruS7i=*@JV3&FdNBgyR zAZl?$9w&d}gN$8>S9L&Z>h{USY&X$@4*z9Lp^VMxu75|E|8GF1L|iy>T2pT0(oZ=@ zXg1*NSW#cuA=_GDOub9QvJ2Q=4w<;S$c8k1kCyae9g(KDY0=J|dm4a6!ANJ;pk zz?(Uy5YVWHo2LP|R0gyR9{VAX)4{dgg~U!AE&|*uxDG_!Y)El)r=MipOBS=h zf)s|e>DaF$LaqMZ1TB5dB@0*E-p6=JSlz8WO1GLZVw#uOFr`XTB=fw>eT$6%#A-5g zgmjwuYa2(k+MH4Wr5}4IJM_aF_{YVB@8ybF{to%)UixkoO_}>6WF}F1$!O6nwa0-6+;^pu&_5z=ty^Dobr|K`&EgGrpWZ+96;d`f45DX)bL*Nopz zd4|R~k|E<#%$cvtxj#Cp3J%cZPd^5LWmncET9krvIqOoaJ|*{uKR}9k2F{5JfjJ~K z>Jvi7KiAw&|3_b#IrC84OeBAkr1GI#>=gcng&;;gXPPLl`IcTJ_4KaF)k~$*?YA<# z+favG+STuVu^lZQb?dTj7!J(Vo}d%Py&)|%Hciw6Sb`laiUAFpSiwnyUg`M++U0CCq-VNt zrQXJ#BKRbCo0vQ@T3?aIiOa~?eN3~H{MdLVnWu@Tn4N(z=sd2Ds#{qC>onB;TA4viQ5S?}23 z6mW+3#R*DTYC!Jcn1+O^eA0A!xh|uvFbp7p;QsX|eE+}}&o@y;c0;4AuqJFMr@`F`rN^({$Gb;X&v`=&LS)azbDK>|eE*;m#yw41{0Un}*e9Tx>PC$)Q2q=LW<(?$!Y zk+MkUTVle79TCExyY{cAd4PLWr0xPg&}xqY2LyyJ`xEX4#&_k#f|u8^^RgG+%m3!f z$^Y!&ysfEAKuBq5)c>_cYV@^6$|o1tosc@9$ku5V*%=BKeEKKAy9E_BQbUx$mPAe1cVV1BMtetRaVEb-Me@g2QUXGbgSQ zxH&3|^<0-xI?2xZQ$iuj^onV{cTcXv=h-h<>;Tdl$%ap=nC>?Q^xnMp_w%Qm9!7$o zEe%P-zTVmC(T-{Lx|zl;)z66~FViiKWuGda>#>yXHOp=U464t6B>v}E>3@YyWR4Nq z!=vlT(k=6kky!&H*VU+^O-vjt$smd*!avJpC`Upb{be|;bb-2A)8Bss9-pUTu&V)+ z&Z);(6ew}Xl6gx?>j-yW8y+!|mEex(SeR~qzVREeq0zK^YxS0>jXM5$N zdTLBtLFY#+;NVa;D=0bc5s`^R*>7u@2K|`iTN(|ljPX1UoH-sG0Pd)Y&T3y&U&Q^d zeSh!j|NnbWA|$&}_$%_eiJYRl<5xh?F0X`NE65MC>dj0awEO2A6D6Wn*(gM#qYeC| zCROQ!c(?e~(|Ce1)T#ijxwrcuL4(3bh|Kl6~@nbSd z^`dO63nDQIiY;Mhxh_>qNf8({1&Z znZ-xaVFE)`vpxre zV}Ra8Ba=-br9Mcf5&TD>U5dnFbcnQP(}a3l}et)Z-B#U2E3+xvW9>2Hz1rg zFtkEv^fE~zHH(K@v6kJqIIQJay^H!< zlnKdsEa|5G_{K5rLLT! z@nAt`%?%TvEmZrNM)jACTirAnwiLJj;jlN3S6q55&b&A;z=1ss8Qf~e z@devGAS#7Z08>XE((UwK0xzmTXDTY!uFBx}0hJY%Lqz8QwY*?&DLY#;((KW9A*01_ z6C1M+kRF=*4y&V>Y(l#vnjhRy#V@v0i{7H$Ed46;}6#J%<>j%MRkjQZE?CoB=udQ(|rB{ zkgxO9Y}T~^CokE+zXD|5FA8A7o2~_`r#?r?ovPWOXi1i-Qfxi*Rhi_yyAgko>$6y^ z%lS3F1>LNHN3vaL8~+8tU-93|fjDnf(uzXsPuh}t_kkla21W~!@61vUGl=Jt1|zS+(BmDEz$mJTzV z!0QSs#eFfq0TJp!2klW!rh1hsTJkcW`iY{wpEAYF&D+{JPGm#lA-p7WYNgIR&UD%^}Gm#VKz56 zasC;n$VltOT5ZHs#@)9M+AH|$7#YLNny_1FVw z?ri`3Rpq2^=+$0PA*hj@?(;qCh>9l{m7eAFE$H(eeNl9cTVz2f^(klE&iKj*JI7(J z4`MAWV@w3AZ_5IJF5mGduKwebH(`vl>Wuv>r)ZV| zU}d0}MuRNtA1lQNqu0&*6p22JxtrKqm)2FHUvEUcHLclKcnCQXf_BEd3rwWCC!Y_! z)|FIc1*uMt=O1D{{*Us~e|eXUabq+>Cb@oxg$Pbjpt?XR(DfxF}XdiC24Z)_huo z=VdM<0zd9T<|EYcg-yH#$eWJXe=ra1+WSgP1C~32nBW936zx=g^0SEjMo z9ooub)TpI2HoEx3$B{F47X=f1mk5l++10`dNafEDtV90)OGZx zPeqC8wrm?K=1#SbZjA7~Mb`t|ri3ckZG|88aZ7#Ja<{Iy&58?lP~ZzI$RZbx%&Z@c zV@3#O{(PQHVSLaOjGx+hs85pOPHBqK+?P> z_cl-2Q{u!$wt2)8L9>&PfaD7!iEF`ze`Yemn1T4&qV!>{o;$CY4OZQ&lB`^%E4%`} zcrw@eeW30B-yN=Gh3x6(l1Gde^CP#rX<-tMsrqstIP_-GI@HwM-Ho6k;32R*J7CI9 zy8B_{;BL*LOV>3-c^@#d+|Q{crq+uhVdXQH-Ri5bzG}ZJzNUW0qj*p+t1myRl(%qa zYGHA*Eta`E+Ol};u-~yPnR2eoD^SP!@Q%>I^*Q&Yv4bR=2IucBW7;gGQkbj5Cj%AJ zGwS13`77E&Q~7r8&~9wjTxP?@3gd(NBpZOG#8Na&G;I7{yhDI8sA50|Ax*MrN9Gx8 z4~Un(|Go3f0!XW8|yia&8g4%DwfP-b0|GA_bn=TR-xNaY$}?k=7}jtO}z7Y z{SC~#6A|v{!sHnf8~gFKLnGbHrfFx4@9gPa7Jqf1jIa6vDkQXj5A740njd5)uF-C> zWjkpJ(PyIG6gDXHEXzclYn=~|qkFx@!`rs+moZ9D9y0Qe`A<7=g{oG(Zr5?$AdOb0 z2w%JF?24GJw(wmpER%(nQ~0{lUYaRy9iC5mjRYc51C<+)PkbY`FiVG&71M?l{mcxj zxT#E;Vu~T#^qAs1a5CxbqiMYSHnJ?MPm{I(ujGxwYm!SDS5~?H<@Wm8&;6h7e`5H` zLGeGsw!eHOD{IU+^T$(R$id5x@{qkxKH}_MT?60t`YK7kv_+id&}-F!BNWQG@udSBzWo>P9p$JsOvsH*4`l ze*a!dtTjL_=<(j^0@KX(zaU@o#%Xo(e-Jex-snhBKjUK#CMKf+Y z<`-;*p_l*W@rOp+!~B>;e2PZjTz+>gv6coofv&gk!Y| z$UP%4UmWqSFJ@=q2fgPJCvO(GRi&QqD0kRA@-1Gj?a*8~h$=Y~I){DC^q!=?c!ig1 zR|zLrbl#J-viC1{))yb=eVF@vUXZ~+&SOZ_1y2EP1gzl$a(t5?OS#e<0_|PQ^GVO{ zei9Lt&wy?*pk0BNCXgqO{~SU2h8}U=I%NK%SO%sPtQ~Py3na`2)8u{3!PXdKO3Azi zx!-sgpDJD7E0<>9GN>}0+K5cj=WNr3S#4Ro#ke=xZ>PitsFUy9%*iSfYxbU?^`B6- z3eOryE(!pvWF2;7E(@s!k+$m|AyMfHoU5EI<#?KX%2{*-X{WCL$)IPaRgV`5Q=CPQ zDs7K8oIM$S2cXka(Yoi>W}u;a#&RvZ(^%EshHFKDcZVIhoG;eC|MY>orrkP%jOSV3ILY* z)EXc12o^FjZV9Q+^blM;3jY=$z>7`mJP>QB(dRtzJ!>(mm036ExX6h=F|D8Y4m6%$ zFWwm6xHLSJQJr=d+||odp=PBF*-t}cMka?}*i~;bhzq$iznJb)p+r|UruGvqF+~#5 z20RiwA9t=6u_KSpqjm{vIr$CvhThIi?@C{K*!pSls@f>EK>57sqZClPc{MV^Ym(wn z#|R^b621oz;3_4fF0~M!>W>kt+qS|w9 zJ%ct~)>gMJIu`7Wkv7NLKKV2Eu&t-EHtRJEq`$QMAUv2t*VDIOlQHdQop}nvY&OTvI7o0oHPlWZyWU?Vcq%_83>w-stj$8~%j+AMTh zeM@;&=SEwU>&IRgi5s^vRMY~&!8|rn2Vm8WHBL*<*Azb{0s3m5x*#vm%g$48phy|I zRs9m4TZoN}jI-+Tmo)s#-EsWCxjWPqp}Gi2yyKuQ#3DfHKttoIEAL7^>ZmWsvTAJ< z=OFF1q^e(FzL>mvIDhWuc10_*SEhp(5N{Xn4>s4N$V@r=3Z3VSwvZk)r)jTW8uk6m z4c_S9%;t6m3$l}+4K}%oX=9Tr91hbeASuPK^3q=(ax%64w0wpt0qXJUstySD+RFS~ ze#88%R7gbqDSDk}Oy}rKZ_0uvIElTpO?7k``*!5 zwvLb5BcAxNMt^g!)pw5-zf2X`=F?mc!_IoyO?mwc$@63V{1>-r^CW)OzRw@=M4lZR znx4LrGM{Da70~4CXLwr3hc}IX^+X{gyI(cWg2<+4hZ80x=-)=NxNj8xp`Hz-a7B51 zP5jHpKf2Gv>(pl9?b`Z04p}BzI*3~OzO%G3Ezz*X%5Mb7q|;uvW2}vkeBN(B?0kkL z#Eh+-E3m;xO~NDd^DnW=uCiK1#n=q*;urlJFKc7O_8jPCzuK&_q35khuTzpgXTAU@2a^W z(9U2@uo^JzKJR|_RnETO9qS9#KQk{Y9BsFy&c-w{-kT|1I}^aMnz&*tu17O@vzZ^n zR++M-U#t{y_zfp&@6bZ(hWiarc4Gg0gw6v&>Z&~*85gAJ@SZ=Tb@}sRzq9VUeg~MC zy#%7=E-Bly|Z$6vYfZ2^pIBu?HnQl48wZi3;8LEk5i+PCN zZu3QcIVKN~>oI4X^a4}>9b4-?(O6_G<6T%B*n%bM!K>^|?mTk@~#g>@3mqKLRg8=z4&V@j2JWmHlaH1?)VFOQ1{- zK8tcnKzz-+>D>Cwa7GTzBY4slF&W9rmBH&A~-R?LjMN zHYc2Qjwusx{BYlk*us|{BH7f$8;f<^*c4p{5)+e)XB!dCA$=p=J00IV`wWKT=TeaT z!v>0xKUP2&zVM~~5(L5#A}j%6WvTP7_Tf|Yp-x-=sf-k>dZ!DK+DX5f3W-s@-S zqtRY1u3+aP)~*#(NALKNbHEI@4Vy?Zm3u5+Y*YC~C*?-4v@QKYY$v1F zcIa+!XBWjCg@;zAQ^|mJI9S7Cx=eMKA zg&rI}{EA_0Fy^Q{K{q|9@6(WH?-$%mnGrVdOY-un#uvC}_%2eKBwKB*q?~)B=?WVf z$|}A%cCkyqGsF+8pKj^!+abgPlU!{UE*=a&qEyT&Jd3-?FP`Md!ZY`F!v!;APTOJj ze{pr@>9qB;XR2>L@vz4eJ=j&gAd)=xuuJ~c0XfujqSH<*X%-f6Z$vbhL z%F$qH7PE{ukb5|M~Ko4 zSunJ%`_NKoq4+@+ z17G-*Dv!4m&-7+4yso7sokIyq|I#(%!PNZN&LUs_i8EwG`0i?cyZ%6dW84+JyBUDx zWw6!($RT5Eg6r##fmF=PbB-&pqsz#VmYC||oXyJs6Tw#&=xKmaG5{rTtT|`Y62Wuf!(Jy{$wuF2)6|UjCPK=4`5lgs0QPo7cyzGsDLb zmiR>VU+o5Sz4hz2#pmqxjf146hXlto7@^XYCV?1A_u)-Xd~Tjc0r2$$@xDUW0tVxW z3z_=6+kbk!K~rO+vi0b->BFjlRr|QK)N^`}jGRCJ^KNg7FV_JK>!uIROxDdXEXY@l zl}{r2oYtpylS;8p1BRkHZFEw%Bk79eULlC*@k~X4^x}vSrw$HE%>fgb&&kzLMot-g_vG? zw5UGZy{w^wTBkNj3G*dbFI?!=anio4UsBk?2`#3c5bV@NKYO9;1?sALsgUWj>E(PR zc_*}Oy2{q};WF|7Vg1te@bG9IwWa2qS1NI=rZ?rHvsIW=^yWrAI1(lHv0xQn6~*`sWmU05!oewW4Lf2w~&x)Q;@X{i<{B|7kh1KG3`_o4NiIlPDC4Xx%&3d93xN zU-T}A??cZ~eftyH#R2kd#rE7yJ15a%ZIfv|sb6Cw%SlNWCjChtcgUspqcv>z>G>9x zNKGz%#eJ-LR+lFS>U($>+&Hn+zVeI!0X0#CngBJ;yEX9s=?~fyaGAKaTwLVl;Sl_er z>HwpsOM=p;i7vF!T^Q6{!fMTI`G=Q^o-e_QzX3LF{-%?d$Qik&jXMkrUo!@x5}pz< z_Cm-@x{s;GJN=@&k2(DWyG{Z*n=VcA1{hnMVcqVn-KXI1kB~Kr^vzE(vp;~UVnUgOG*l+Mr9khM zsO2g164{8MeLiGD2n?@A`Z{A9`Vpr)Z*&{S6Kp+Xer0vY95**M)wX;*l77f8;VhGc zy)!vQt#~kYDNr@{w)`kc@^d4cbDF-}5C~M|@OT{C!Q(1(jf8V37Y@><){^Mlvky3@ zeKzS(5ty3aXp1iz??l{;-0RpM?DQh^Zu(g!upxtB5B=V+!D0Y5aVPyN31@M`9FtUKT zaTiY)xEUB>wHZAZFmYLrqOFdiR#h_J?0QFk8s2O&s}f|WaZTM^85~>LzWM#pV);$6 zK~RH9wM`fFrzu$yzQ82mE?X?hgZX)UmY#y(Y&z|eB<1%A3pe9Q9^}@}=Y@liYDv** zaV9-iRHH-l%bz+5pWQOG8SSBNf}0UupEpir3%JLVRWfuuMb5;`f$}dAWKkk94V{++ z2EvnSjkb*#rLdBPontqzq7GEPLs?No1k^+D$Ai%p9p{iA8^A`VF z-KGfN^5YG3yOP4pP@AhCd8KY%ksN8W$oSWZY)zesp)R}O63*;Et{Q|4w-(E&%E1vk z(K8_-q1*4XEeLk>5?$@}s-$cpX0Omj=DWh9rw)YOf3vpoS`k-TCeg;KkC$cZE3ZvO z^ry^SA(@WeZ=FpY1>eL^3x66VY*A@(*z0%z1YV>;Z;)aKkPc~m5KFonXC32fH#dUy z;zLw!-|uC>mjR@WR{o!qTZlV0b#nQPo25eIo`Ikkx@^d#i?9|6(p`c$iR$252>V|O zoHl6}bsF5Cd^;9}#DfknUc1#)E1gJ-m*7zSoJaRAU0nNgMavmv?{B3K%Zj!>qU}sP z)Agr!ZW+43Q;8ZJYvEx*)e5m!H)NLF8~0>rruKzh9Ho4s4q7M|Za)sF1P3j_=P?wz z%rZw!bBX6@rcWnzwym)?B>wfyto<(0)n!iE=jAP`M%S#*%+CHm`kq5 zi3a_A+b86HjrQeClAaq$r1tziBD6Ud<&Ht7*g5+3KC}8g?vwHS!wFb3`r%}@w-)wquVB>+c?;Ef`7s>pZ{=M6>(c)XL5i-0l8S2u3KF7*$BT3jqpN# zRl)6sjpsW{IvYGpB~L)~`YGo<>~!mS5&fJnTv_BWw44~^DzSWx3$9s|Kplbh*cR!aTd%+#o6t8rxz7i?_+t+C$*k@EG|aT z)HN9TwT#RSUFZLfq$4^c08>VAOECty&vnF^?+j8jS(g&t-y0|FJ3uLs#NrJE~v{V|R8* zH}LtpLU`(kRq3wjz1dvnN%~PKc8(ZTh^mT{5xyv$cUWQCMP1g_%DARde^@{naAq>w z)Oxe$wWObX3cA26kYf;^r6rNaM)mONx6gAC5om2qOW^B=f_fnG_O7S3tP+U?6qiVf`n$a`J@MMQd@XI z0_t9Lo!tM@YT}`$4 zc`kgfk{IBJb;V6m_?Zwe@02*|V;7&U^XQfcb3N;;NYy?2R==qYS_bRVYZTW?vWuy( z+TXC|JrZ*xC;+x3(j8S#8u^7o#9ip2n<}<-@^iig*qOkUkfxb{fwzy&gZJlJT__Zl zA*Bxz`u6%k&Ji)ZL}N2$zPZAOgF1>%gCv1|C!xDsg;FZ!>p}`KwG&ku8m}kXU1D1+ z%PK>X8ny5h6bvCZ+q#Z3--$HzSE)3Vx_$VeIG%a>8xVJOT03bC;Ulat@U1FYTmcI6 z3kqH_xJd~W)HbY?_zEHJr*3YOW-{B?-^rcnt)ez7YQQF84C(re6f166Uvya?;xM$5 zbAZ$pLuYYt#vFK1{6zTJ`KS>$zI41kkw$YE^)wUNSa1wKQuxSNlQBU=<81ouycO-J za>d>l#PdAlZ@R}yy+?kr0_m*u;V0!AKhzrzK4e#%0dt-kkfIp+lW^~E@ghlv$#lWG z%II6i+?om4(+||6^&%2J;vh6zBr!q3jdbSsH)8ShBh7InOSR2X6;v|&@Lx0o|M5YV z0)EM}=CLM6SMTu)A<5w5zv|(4e(JY-uyCXCRyHeZ1QFloN1(45AL*HWeE5GkSlfSr zJ6#Jl{l9TNb<-@N@}uj^my|n$M-juVZQ&OneKW;Hw6tw0)-I0UEcplHgU<3=_3{f3 zAus+Ae_8=${X)W(_S^y`mK z5}k*v+DFqgrMg=R^~NfMMf!6``Z$l)wAmvaz5`v z6DQ%64BC`9x~&+c{tE=8r`=xMAU)79y#MLQmfJ>V^}H>lLukseO6*l#Y5%1I=aTBF z=SCK{kMM+;7((;p)spCq^4GJTy1Hu94~9&H##M?oOa@&O((7r1qR8IAaO|n#CiF0) zX}0Q-&?u9>Q*8AuONoJ_`sitgoycP3x#-ime2!zshy_f>WSYN`8@}2gu=6`YGCywo z=eByIIPn6Hqu`1DowU^n>C#@|Ebh1G8Shasy%MUAyCbqHeA#8-4or)PPNMqDNSi2p zS@FhVPl3rP#V=mR-ILcI{?qsersJhhu*%%A@^NGl_p&pfY52~ipvt7W_sb;?`fV-K zno?7>Tv0NHurEcz!f!Hn83{V;F7)Ystj?*VSZ&V#9RK2DXNdQxOXai&A(c)1mabF6 zg$**cGoaD(ZGC>k;2rI%lgA5vN;{nI7yDE<`o3?G!N2P*j0=)2S5pFFQ%_?7|AVr( zj*II1_QpX{X;Hcqq`PZCL`oP!KpK>x8-`8=rMtTu0qGb41!)+%XXqMW=q{hp&;364 zd++@{ujl!lKhBvycAR}??X~w>>%HFFm9n2w)0fa7WDqev>Aah^#pN__R6Q(Qyr&&m zq3nEJui2S;d}TQ(J^~PoCaCz5^eRav#OQzTq%bMN|IAd}aw)X|iR3Zh$hzhMXJ}aH zP=*JjF2pa_{j3i5Pdoq>l#V&VIwjfWQr!It2HM)!7Ebl5zKk0^&iXDG&$|j+t9B(7 zlnai=pm;82;2z?EK}-A}XEOZt{QldmM>LAI(TIB!Ud~dSf8uJ^uL>VQF&hawaHC^ad69#>wDJ1&Hi~42);n;lN)T@w4340}ewGSx-zO7Tq0n2p7_6qdOa(%c-j49E!@NFh zq6-Nzkh@B7@2+;IkJMKch3e^X@xaE~jZ*H06pAQ7Pkxnkc^+SCBv@=?#QMVyOf);B zHq4ZxYGY@og-TmMyqAlm610q*ZwPL1=r)+tRFM-H(G4AEF@B2(c;NC)XV-Hu_~a>G zkl*Fd8oOJqCe>1m!fKgR+^UCD!qnMsv@?av&Ellit~Xi~FoyWReCGb>FT+WnBX6B# zj12UE8s5YDPUFTo1?h-GJ7AmVfu>X?ON6G=qhuSX(021m)OcGH2^#@rmjN|w?GlYr z1D2dt!B+=aI82J7kfBhQs@nX8=XVJ-9q1>C1w8*u)FM1Mk zd{@F-6-$pqO!F-PHH9vVxBSa;s={_Cl`JDWn|sH-+%X>^=l=o~D=R-Al>R4HChxUo zsB^?|`*vtrDTr7D(=Q3i`4wg+U0Z~arq+*uz@(fmrMEm}{+>J%$X$|h930}2@&^HV z5F1vLT36E+;p4#4R1v*_ng2&F@fmf0qj_B8U20pNC=pw1Gdg<5PZZC|-?WW~PY(=~ zmRe^SsQ0}-dknEt|JEEe4uDgQO=f>&h=WOqzGU+0lli=O*`I&9F{rK%7s3Lx=KrK^ zH^@mNib@==Od1_GU$>X+r`Y`Xkl|CNnVEnJK1#{j;Q2&NhmX9mu`ORNC?+zqunD84 z#>igo5dnu%f-|Q-9WbF!=0-a-)=21fC`A+*Z_b`I*#< z{t22(5Yd0S&Ts|(ahz!(6OAfp4g~u7UQ@OWt#9!_e*oU^FEqKHJ0JX$I)&2ZJX+q0 z))h^8YtaTBi5dO9!ouSAA<4Ay*myOWiI1P$LSOriVrIzixkJTsPye+@{+Fi=RwcbQ z?H6c4qnt<1kR}m_07kmONTk^1Wep@-3hQHT=3jG#W!HqANZ;Qh0l8@eMXj5Ijj-E$ z)Vitne_1!*>SK{W7P<$eqJ-41zWc8YISTwn3v!8`3%5X$fY02beCLnOnP;m~e(t-{ zS3}+%6cr+n>vzBcbJ4<3&CqUMD-zs~b9Ho8FxMWwNT1{%n#FOD{*tE$K+TSkYGdy! zL!q^2$vw`IUIc-a(|rU#1Ue+Z=MHxlJeKS8<$fg*{vp^S3A4JajT4Ko@eP4CwVWz|Gjc-5-K2goVNEKo#N1V|Y4?RHJKN5) z*k!t0DKRP;+YpV>AQ-*pyNz86yMZCOzVPjAzLPLpe`@lWz9X=9dNF*!Q*FT1t2YPw zZDkAN)C12ly}a9wZ)RxGV3*y(^~4V9jw6~?*++>%`2*_I%ByEgiY}lXE_yz+D`juK zGc0i1D;jRq4VzBgAR!^Cc8G)e?%N#M_O#O4YAYA+32}aHG}t?opQ1k6( zeyBNKoSYb^R*Tx|MA9x+&W9 z!K^2=cA+Te4y(@-`u`Vj^AnP31-_xLv%1bYtP<>SjTsNa6d#-mRO}e@u_CG+Tux@+ z_LO2;JF0Eeh!tdb%6)-Kcq6f;*vlYTEBv z$ws`F{lz11S%=A|dt7;C!>=Q;_4VDAIF$J}(~Ie{h)q4X0i({0Z>^}Z8IRacIytoF zAGMc%S2vEzNGfIHCK;cqDJV@lYD+?o{Vpj=^kGisgZmoRz>0=tMue|Mn^WD}?oOX~ zc?AXet=?i&H6wtA$?Jb99N% zmn#N})+Ypb&&{SX|$Ug>+@fkms6-w(@)Nx zkW)dvyPhRZPThDszE+T9Txfx%DDCVmQeotM2h1388tSg$5S6qWIi3TA^z5@|0LPVQ zNwKw6F&RwU*{@6(sm|l!zhY_Rov<;?|I<(NuiZsDANhcCnpM%33)@BN{Oj4F=0zW} zi-qV#6!7wK-yea&#bpq>bp=}i6=-|P^*MYh8@Ni#K0$Jx$g?>5$P+43GWg*#TXh*I z;esM&DPW=(jI^UZ|F)(0x1YTYF#8lWv1$l4qwf^k$;Y@zEqQlW#)TXDfSiN9Acnts zPr|o7hZC;Re>}G^;K<^Be{-|nJB*AjWm5mYr$t~pmx!)Z3r0oiA;t4+<6+Z#(d5En zUR;;*TMf|nmuOCaS-l9#c4;EV{3ee#K7wHr!vg)3-Fr}tsE!r#z1LHcTY6ExN2Da2 z{8d#xUkuN*H3zk|C5GzMBnH>xw#vYn-tqn~Oe?A?l7Mc|I>|G|O8xxdC-V&|nn1q7 z*DfLPm+mZrjgU$2=SuV?2My=`pr(FW&R_B;N3ND!EC@!9xsV=H-W-@=x-SgUT;mC= z#!B;_u}I$(-xR&*rGHn(rQD5l{A#tNUUvZ8-_n2IX7D15Fw|Zd%Nbw7gT)k z>Aa)&3O=fs%TTP&|0$8Afte@qi9{A!#^A^|anl!!d=B$asH0uN-gWbN_(RTIOI)_L zmY!Ecg_snLHRDax$j*MJ217(;1Ih+YDz1&1(om!)E#Ts^tBLiq3xdEG%8BUZY)9IS zB<`!#exvCJW{1rL__li78Ry(LU`y~k;yF2bKx=aY|1uxV0s6XEDy*NaTjV;F*_5)? zxW6esP8%5sS`3=zrrZ$~>UO9$Kkwu@OWyI(I>~sU1S-o-wl;O-8L`Kd%GZQ+RK_^T zNNi_$G}9-hdoULJtNraUc&sX->;A1Z{0A{nKjIku0EKb6#@Uon+?(IzS`ebkbXYZX zM<9Zg_5&PpE4Zk3goUCe)7>hqeppRRe8f4L#GV&gz{W0|$TiDO_x4|3N0|-ke%>6$ ze#eNVGyj0=$tMj6<)f*39SD49$z*-IoOLB|$6KcXD=I;)gfiJl4XG5i2@Yes9!6csVKkhB$i4KGf{zB$u_BzBt zGPL#$YDnk~*LKzrGHh>@iX%8lvVu4SfGb*WhQ7c*7JT?n09)t~%>m28utY{vfWInVleBcZH_G^sDHcsz0joJH z=uFyBM=c&GX$$hbGA(p{&P#!d=@8Wm@Mjl_?sK~Oa@FBd7Bpkr5>mEr3$)xA$`h0e^nNAT? z0Nusdo8?J*1RILuSZ!E^exagWN&$f|FYdiGN0|_Gi?Y_oz36{CTsCH5J6RYjR419-DcxjgK3XYh1rD z9&DOdko-M*JWewfSB*s`pqz7JAT@MfhvGXbqAfQB>{~M&)-2(&a7&@LDHXl<)p&Cw zHYxl1+MSeMZ2#0FQS+RMhrwwAx$fuR2Caexc%8nz!>ESH-C@@bmQbfX&}% zub|V5FR-;;*2f*_EZzo~y&IYF_?3hDPdQQElXTMnkNGvjnA(F=9%-b+etpSRL%<1; zj~;>qS1H^(o>HuOz^&CzY<{C{rkhH8%-hEOM$5Q+B(ak%A)!Ry$=>@LO>2VWrXBUQ zx^@u(d1^NrDD*e|$h*HS+r|ZC+~M4n{zlu6*{T5YNkfoAeI3L1U+SY*W2?VF>c3e> zt=IFH_NeLwZh7FeqiwtFo#SQwDTnfXte{*`Dc9uE=O}Dlt8{jyGs-vOSLOD%0ivnb zL3e%k-VMJlRs3?SQHB4!7olY%n*j1MPS3D9rZ80ej`Ra!azROt~^RLDEKNN4yohIX`)MP8I7QSvxS#P9ZU-N-ttD-rxzW6!tN5`P9 zu>AGgq88-w_j@_1dp*fL3nR$gf)BFyp!T0E5mfd%cY|$CL3O2hpI){V3qgxmcYf?K z{A^qX)9$=3ZVt1OwOddxKyZ()1MYforSAPtyMG)1x7+7w{AQ^>xvEBLKa^%?-@W zTq}Q+n~%H8_|n|+uq@y(y5wpJ1Bpsg7MEy!$n*pd3nqHhysJ?rscV~oA6@DeT}3u# z;KA1&l^o;bg+uWGI02D5qV~x}tm898m62Tghh{5?#7V&?k_RKZvue(rkSTPPXU@nc zk9A*mVvl96`~d)iGL>4j=0rWp9R%N|+Oi_GwTa)LNidmlgH?%CKR*cC?Cye#nKq!( z)AhEmoWEHpy|-OYnmx;5>>EU3BDO^1SCyFJgG_c*#SiHfY~EB#`%kx$4SE;G%4x_t zRTTy58om_ubf)$u_P$lOD>srZsj&V~R?=i(QIZy@xDkyDo2icsd4P?}SoI3GHSN@# zxhi*pRMSAuuR1?_1*ovT(E7&Hg!F~KI}*Jr`&poihbs;a`Vsi+n6RIY*@Ski0Ylqj zURi0=SZiF-6n+Uw8VuI~d(qCTpKeCwOh$RwKXl?cK~0yNc!90zI#6$=Vf;@)wO~fU zyh6cGjlwN(MdXo;#PPenfE(tZ%ld$08QRijpAyGp@A5Yha*X=tkNZD-sTsukab^~^ zY#w9%(>7fO^T8t`V;Z=P<9DeEa`8E_hNQ+;{j|a#)P=rW;AkAx!bHl)e@vhUc&zyC z$rXe`l5sQ!Vu4bsF1LG(|G5lL{>L(`f2^-7D3zbw7?+#3J5hfJ`h+4=(CU1sl3=HCCfg2-1TD(%OiBFU=P^kNEeDlY>BhUEqh z_EG#vV~Lcae{nQkmIH64nud;ZmK>H)K+xtj2kC$%z&`-a5;J0|0AoENx!-7S=j
TcSRO5{zd;fB&L%d%n}kPWXzW z;sxyd8}3RZxiodH^Xvn)m+EGeztJp5KL(Mk9FU#HJT)rCPHg+>C5%@%ZIJRJ#xbQL zf_m-jBMe)f4px8mQ?YPWW(%Re#2$Qj_GUhIG)U_FTcrOXF`Lh|%?A;1CN0mJEE_-31u4gv1jml*9v z|FUNv;44Hd+1T4&#y4Hz-<`L_l$bw(^#!?;?$$%V8J{9!e^`#w$gS*`tsJ0nx_JyMTTh5%OB7bu?x-b9 zJ)Kgmh|k(Tp*FmbMVqT7i^UO^vN+IUNE|;Omitxp{E$0H-!_fS(al>;OGeLWyeaf^ z6YFa8XlS0j8JZu80@g8U)J`HVmY|Xwm(Ku8`yll8{c=@ihb_l~Tr1)iKnxfVUG@#m zqb2C}WazV3>ijqpXDE&`u`P_j3-j^nC(w*|7EE`gjn>@WqAEwu7U>A8o7N%a((cJl z@8`&~8z-#on`s**zi^JJu1uuF2~qxyMqcffNj*K)B)&CfGaSEY!_zvRUL1Zxa~iLD ziQwuayRy#nk)5N#Dyr$`Fk71XAyHg#UY~aG4Rux{HeV-wgH-sflH6?&v3coVL-Bd( zFB3Lyxptkb4aCLqqRJm@a`9)|dECTJiW!g(QW9VAsA1qFCSC!D1z4 z7|#LgZu>a?eawGw#+VjH$Dxt^bpu=mNhc;s|sNwt@^hHhOSjrenwn`S>xOV6}52XIUwT>drM~1 zFBH3ZOP2$(@G)OzU=Z_aHR*@3ivX%dpPn)@<;-~(D!7D_k;%pqBkc#i6V~}H{QEx8 zyo#U48bDD7B3k`xH-$iw0}r0z=TKQuKW4zmdth5s);d)y%0`7TQsq_K@QZY>;nyi< z?iQ3-;GL(}YGX`eHl>c0fqeeB^Y&~Z#1L603vL^g<&wicZco=C>lYv}poLJwwvQ&= z@B@OpgJiXBMhJB(EFe|iozs7#x%+yrCO2tJwl6c04fJYHOpc1b>|=~BiUjEmSUnK7 z?_-LsT{bxDC3b61oN`OAdt<%Nr0~_Ec#^Bx_IkxFJ6RmDpkeKnG}@XK6g!Fn8^;Zr z<;^*>vb3wo+KgYXMQY7Rz`U==dLANqXRcjp#*|(n<(@t(h|e4_ z&C<1>K1$-5yoNP>VWOt=tJTj|q&PJq+ z6pC3T2SR;9m%ul_pty)|BLSdQ`RI9{;E3oPsIass(qL#9Y{9#-nCIzpK{B_WLj0MsSa<5_m?Y)2q&W)16Bcv4vKnvKx5u% zdia88#jF8A`-~1PP+R5mhnil}hu}q}u{=b^SjFMtG9p5Xc$^E@1R8>f*oOK!&e^ zqYG48lFp6|*uKM`osbB&{Ol}#-m~o1l}R;mDw^B>;Llz*lE`u3Ijk-q2`ahJ1Pa&( z)ZEocgtz)<6U@ zNX38^!Pr(SM#CfNDH1_`30U%_ zk$%sorUx$Y<@a$*wRl_B%G9Nn`EJ=C$40cbib@1)j@L^!A9_I@cuj`TZ(@vo&}^0C zc}L+Dmp7P8mFB3;7f;rYQ80Y(mm{rs9vO&7&F665Gz~!S)kN(YV?X`{U9uD3Kyg`qZ|B&KD z9M}n_<3}9_;Ir(z-{BH?h4Bo7mGb2P_A6`|nXMO+{F(jKbT4}hN7a3=*aKiF{5>y& z*S~(0BoyCu@|--tK|Om=&qx0%63~w&nI*DM5|B#l*Qq$u44G_L?NANpvlk}O zyXOkO>V;m-$sF$zP*OWf%R?8F9-k*IEAnj9$KQq!t~eGHPYDHL6YfN>g@0C)R)d!Ok7yA;eX~9?tD~xR{jW2z;io`qG5LdI%jgr=sD+|#g9Jb zI9ZEDJ+;A~0b^bvDPYFfhP9NCW$!}bJ$o_6JW(0M_9NiW*$jrEl^bT`|NX&Ul0D)b zhXJF$iB!IB%Xi=OJ%|ZtMSZKMrv%K0x4>p`otU}D9tjHw$O%!#e&-_*wQ+vHM_`i~ z-&t9TXxKv9jEC<906sT|D@ASF)ac%_-4l=-*iN z%WO8itK)Y6g&f+2i2eP>VYpJ=@kw>U>Ms`Om=Z1WRUACKS{-`$X5HzQi9N3g1DShO zjX1#g>U_mj!0tHO;j1|&(kVC$kpGIvLKgwbsuKjZ)k)=A9Z#B~$P~ zz7rpeN&GXJWkERFN7?YGKwLzyO%b$*rm1Rg3jjybd$Dp;TNazYO#8n0K6COc`;|zt z9Y6#`tyZ@eG()))2t%Zks`Ym(@WjqLF1T!gR27Wop#CWK*Y(^M`{Jy+u#X9zsCsFa zi+bNqp?vi>fVw}`#^$}@G<*}J0{YvM1^TuxV7dT1jE)ye?dLTIop^bc#iS=S+`rNI zHrq#-#WLGX>}PwHGWNEr1g5Y$`ormY`cOR%UO6s+EiJ%H{@|P}q#r9cL3ZjSWoSyO z4VK#6QuO3bFFj@JtMFcr zZ`NR`qUNayWkuW4!u9cql$zm1<{o2%%xGQ4i=Vob7Ts&k-iF;HatbL?nwupRGq9-D zlLa!j;*pt(xy7xxHJotZ62^fz=exZ5`rryt|54mU~40d{`{%Q{c#v`MrDzP?%UZU2i${b$$puR99xK*^c>{8?-5 z&~G#W6<=7)Jl`hcpg}Ge;HIQEwKJHb9urO9-5~mf+PrRN@dvv&U!r;_TLRvv7{| z@fB(x2!<$qNG;*ZWtNai+EvCtaoaHQ)qIG$Gltd}UVUqR{H`XJQKL28MJ+oyzlx@C zW^7N*NBrJh9J6(%9MFu)k+gr~cAKV+xw#0z*9U&GWtIEQgSy+h4$r=f{p7@lbJJ|~ zO$4tISXL%~sknV>gS7sz;s#7V0!?UNCn$E6*;Y=@zeSdFKhSB3q9`huXG4fk;)pBZ zWv|p!4jQgM&J;PKto9r$6_r*pGY?W5x2YsU>Up{uF*Uq>^`X2dT(Rtk1Dk$?3+rl@ zaILw?Hi)j|h`fiYg6#7Y@uXFk!&K`khxu1Zy|Z_mA;c zbq7Daca(ato25~X{k>*qC&JusOKFE(4>&L(f30ZCDCpXtlGu60#0Pn5ph(2X*>~`n z$|pFJZG4eUh=}xzhtwMSwY*`Mx+=ZVz~XDRxGmrXQ&g~UHz^Z=**JDs7Ay4#Uo!(* z5Lfd%8zheZs3Fa$cv{lar&=NgZN{b3c7BWb@xjTSAbSal-cE~zG##1`$4bfkq|J(} zK@S@*b615+C+BruE%apMP6<(Us9Wu}P3JF3*86O;4r{qNy=2Gu=xV`l>Kc6(=5UC; zPCP@$b7Y{eV^tyHCY0RkRpabZB5%Y|YVSs;e?;@HzdSE))T=m|_uIhYJL!%um8wBUC4~y_t<-%!M#^eD{W;Ez-)BB4(KCm4cf%&p#S2 zN@^To0Cmejo0nCwFwsP;u z1(os(OcC!{ZXLyC5PJoib(Emy0F_8v4f*$;hy4VLdzO@dQALLO)>DNZY%u(9P6VC>vJ*1aKcK(fkWZ!P6q#qr`|;WYQ6UXBx3zFBKQ zuGP1BQnI;Gc*?kMaOl{A8TbHSd@%s&eJ>@<;c*hajzT8l?m%5VY$z#d;$InXI?nbG zkH2&RXDTvye`&T7~=!79*fXSznJGC*RMuNR51R?qf7+S5Y0h$fhgEoww@dJFmNa2dK2E-YRtZE}6$57~#WvQ>SvuT9fk!`4J^>_}mr*ru}SsEw5 z0gI514VCq(UDb)PnefBcZ>6m_B*peHKhl1rYgPS?woS*rRCY&V(~?m_Ta>?PXx&nc zir-uqi?y6?F{X%2H#+X_^c_XFpnnr3`BlZ{EUShvW||o(MAQ@Jhuh5AnWkIW+CO1Z zv@SXc*5S{8MNKbM%$hC&u|5Mh{S>ub4KT7%WwN4{1CIClHrfW6011E)1!Gn#`x99O zAuaw1ohdQ}moJ^J{47M(=OK^bQ2ec58`=)#t2j8Gpcm#nb46!G(``)y5!zab36m4! z>PqVK++f~we#=iCp!6+%5kXNLnvf#LhpzNrY~>4ARpW}Ab*(tl{a1{f#?Q--S-FRU zNqE7ziEWi>fzPQqeT>d}aZ(6FpM90oUv1C$?lf*>pk3rhh76F?4mZ$l^cLf67;0>9 z7Y?J<;7R{nOIhH|6}Hfb(vqVjnZj$9WCvQlL~TpE^qn z70-*4DYMNBd@G|WzJhHAAK)NKS ztw}3uW1_x{T-2G4{3RTMBy@JTz@&fVYj{K*DkM(yR<2~1JDFC;JviMDAe(!@=ly(1 zGM=BRL*=`d^^k1*l_Chl1wNOU91L>3w3+eV4S&Z`Trzp~P87-A0;_bt@v91c_Xg`l zXMRQwXLZt#6x^CYf*%VIkIII;R^pcv?}W22-0WNfJ<_aaFEmx@j#`>w=VD#&t#&7x ze^Rn?+sE~DK^tpMjWuc;;xL#B|C}6ZR&QSEfD2X>d$#xu6YZ~07%OZ3K+d#j*vR>7 zNpAiZNKsjNQcW+#Web!64Z^k1;@#Y2lU81~LDFM6d0EwJ-*W~o6iW%0k~l?v1j^-? zMwe94l~(7t3UW%5oNQd&Jj=bw{AyQc%I6uy4JzM8>}_dXwVz5)7$5i5<)Mp4{@TYl^_gSEbd+EgFx5z;oEQ&Jtt zu_3E9O-JC$oo07858jP*Pq`oNhLMmEU;QZF%mnz3?E!SlrOE@F5DXsT?9OdI3o2tWLbH&w2jg^9a)Lx;Ye{y;yzeicCL~ zDYYM+S>UkV7Ze~E(;6R`19@IH3^P47G-`0^20I`HfErVLg@va&z`n(Nw=c&#exd)} zq0ZONpe#q2!QHGeXad|spa+waLA=UBQHSr;8SAf|3g-58{teVdbx0>DJ%MtYX#i>B z``E<@Kp3@KqBrw6?l--!AJg)p%jE6T5xd?diRk-C)y`1qyfG@L*ET7%(+U1*mo_fh z?)1adbrP8bb@pM~h$?aZLCGqNowmH0IW~_Xlgg$nr9n!|7Jq2!+zfwX!>$0mq^1_$ zDD(=6v5FLGMseQ$sPrds-Oq^uDwt2{w+~_!sEu2DrS;NC`>fS^X93{1Q##9emW*eoLzS%3$7;i#U$(w3PqRXr3!*So z!tZl-{)vuISgC%aVI4(pYA!c5>207nTw!I1SQv6MOPEsG?33l)zIgWr<0TGyn-XU1 zkYEf^Wjgm!#l9B*l#*0QYebu^r6*XAwGs^r7A0>LltVbulkh!=y( zFm4g|caxK0%q{O`UR|P`fAG59cuB>z2`d~cG@-m)N$z$s*D0`VVSd{V-iv_(Jz6)~ zfh7$hsq|IrF9M%U6^*+{BEZUYy*>`R3+OoDL!+X7_j!o!RQf=84|jF@e5IfVMSe-G zA*~jB6p)qJ0sMI(b6ORlJHdEfZ<%W@OhiDXscJI-kX6!jbMq4}ZQj`ov-}V>)leT( zluA;!=fmV|D}yOfrMMlmib6MJsF<)F6g6>d)NZ*{|ARI zd;=AiM)_tX1}BD(XsB$mWP_;M88zR@)`GY1Ms2)}bh$nB&!wzw5YP`FK#X=8ooEZz8KNg$UM&#xW%&ho4FbQxTnOO$Q8PTcLbw&k;1H2i8f*;t_tu0@oVyH#8s? zl4e&2S)-+G`#et97X?*k9*;P*S4n`nH)gQ3oQF>Quj%xi=BV%;5b-n3 z4<7arNiiLw_NaFZ!vH5g?e=V{@;5tgDr_^JoPsCRXbzR|A`B@u+l6mhBcr>qV^@|w zU%;$+xTih1WE31KW;~44gyX~8A98tMAoR*vEz$W(_hb17P2UoUr`2N|hJ~?9j|n~- zY2c%Zl8meOZtaU;VXrlZQ~x2m5spM~lNgG=6QD(`Z`r9DHzD#!bPZy|KFs3aXO`SX zsjWZq+Q3MYdNuJ^kNljWFi=3Z-^_>cF$82VfetFZr9~jc4DxJxYNkCNWf2G}s7lqg zKUi?%Fm>h5x243;y(|oaHMPpOnoie0H26%e&(cX4F-D5`^we*E*KtBf#LJ#+q4026 zO&O~@!-kFbIbl3W3fV)NErMU# zp(WnHe%=3q^9%G#bg6H#uxM~xW^Q|}r}elyEkPG88&Jbl7)l5G&@xZYw8!7isXjvb z8aj)S(U`9i9DGKwI~qvILMEzIa@k*0P-`-$Lv3BoSe4ssP;R2E=TH$NS)T5dwyVG) zB0)&o70OT_D@#1lPF}x3yHk8n!IK^lcA>;IG5@S|T3bVlKR(_)cv23Zxcdp`W<#az z`rHs$ejM3G?75A~U7`w5Hrz|n)=$u~u@Xe$pWgxGe7=S&+KmAz!)~(;IenG}B=eZ| zlUB0OclLs1ES8zGYUVt(?E;I}w?!Q&79h56~QYXT^^B$F`WwPr=`;^u`AT6A#yj5XB+xwVz;Jkmxw@^xf4itv@K>w9-|H^`j zuB!>&X*U2kHhoX`PSyzJE8!jFjMG$8856@t?=qL|Cm`6N-p3@48q22mI{#?bjx0*2 zqU^M?yz$MAL$rB!+$xRT!!5CI{Q|4j0LqV}7~(4>gh*eG@ydzwFq`81 zrwt7&Q!zvdY1-OcT0ze$DTvaR@y!JBVBhb;c{A;#a-Zibk0eh`Npe6sz36c zw5F>f#JKQW4EHauqxu=H=-_ppcT%;u08r?Px};^81iAC+i30tG@>NpPH&MsRfTsX` zO}ci0{m(pBRj8yF%Moc@=kcb|8W-A8-xJVMUFz^}G;p)!33I}5Pjyv)K^(f1_|0{8 zYoe^vw7z}s19=(OFF}p+rip8JkIRH_2)=$e(%2~WJ33NnbibfZw(OV2u^}AD1^y4; zi3~}(AFDn6te0sZBnO*bSorvAXfrxDGWFuqDGKjY`*%UUOefQNeOUxTfm|AfJo^}K zLS-zs-|(rktVdgw3K=xf*B)Q_K<;`~+^>lZZW9}>)VMYjSO%|*GT`TuTaE!S8!}FI zi53OfAFxe$*_+JnQc?!tOZ(ow((HKPA}))}AJ+P&g8M~Wiw8BN@r%FF=m!E<&fkc! zi|v1=^!-r|PdJyM=alGxp(se`Fq93j_)4BsuBXx~H-X!G)>1?|zTa!n9iSq+>Xxny z1D5tm?rZ1rb?-d_S|~0V4#o9nt%vBeTp}N9k{x)=u8aBj+>C6i_b6zQNMqj z*?)jaoVxyiMOnhNXE8nT5&lhpI%}TXA=KllqV6_>qx>3-@6lKFn4w!fF8Goc7eOeP zCBqe!#r$)UM;``BWYHY9d`LuF`IcZx(gy=Xjk>I$OqE%*xH=_K_qDMDYul`)#fLJR zuHE%No*fTwgnsxa0&WYFBq8(-=I=;JJG8|DE z&(2~#Pj&f4qKUolF}o%P^0^7Sm3~4uRHLDw&SHo5hQ&=CC0+`=-yj`rg)xCdq(zm{ z#mAe86fOA03oT2a4}%6vumE-LLLBXw`kymuM8*BKlhxFBKUwhF#~Lk4c@ z=v`Vu$bh%?rTIqIQ8i8sYW)WXH}(%MX!#b7a~rANPP4T*d}3y+)Z98B^ju3G0iD2Ma==ycqkX*p61{jozdcbWY^A->PckAqG? zJ;J3|%DsayU_u&J@gpBI&pkZ7p@83LG*$F*c2d17zSOMwWzj-IQ^;{Z?AMcuHfG_{ z8*zo+)$wqvpS4+d1!ZopPe$CEXY@E-p|&;-iH+9#s8zBbeU%5q@EEVvm4hS|RMGib z(=@bd{kYgR+`@PMUUXO5 zs!agtW{7_5l@|!lbLG`(jk(Xr(7kcHFPoW7T|ORaY-3KpRyrQ}`8}h{h-pwEPd=eG z!m&MMtj3OIEvjW?0{(JKpl2AG?Ksu8&?YSy@{{)_i(Y}cEnQae$g-;m%g*(nyww=A z0;{5QWM3Wa^bfc8AibpiA~xV2W5uJQVFO!=;gY_Q&pQ4A;vZ^^snj;2xwJU0MoM_} zsl&P^EKKdEgfsy@b!r_MPxL&(ns1-m1SHV-Ph>3WVGY+AF!gq%4x@bCoAN)ew4`mn zEs`c0jSHnSv+u7+^(Q+rL&iW>(d7*>DGk@ggfI1X>a_pmD*xA0hP+JhOscdY{ZUCj z4YQRcoCJL?{N0a(O@vwZprQdNE2COKe+iloA7D9A&lY!3rwUMZ<&f|wy3A$YBF-@zzD$GVBX zdaIg*<0(DQgNy;CO^>w>k&GtF3%il=z)o?1DQ80Oi}@ahO?A^!``Qnxv)5+g(tOR3 z4)sF`S61#EU>QN@>?VxM^h;d5j=YsQzcMr78fnXHR7(#=`mxjtoD4QM#qd3PmwHON zfST^ctCye6?oz?`EfbD)b*0iYXjjPz@2He@qqi*Vm1%r$mK;)R->tuYz39o8X;~3* zy?lqut*2HhH{W(p-AE0}e;!RXrpO$vf=@{goj>35csZS2P_3G0x&tuZCX-)ih%|2b z)t7L-(Ti*u^VF8mGd16G;GF{PKC3 zuFd7a06WkaMN*NxYwp%RE;v3kXxB+G7uT#f!mA7w2&5T1eL{DvX#>`<=4_sSjc@Ds zd`A6^kwB{hZ?-!l?I8_)bk-XTHaBVUJrn2p)a)5=vC7;;adNkL3(4PTRfKZ{_#?)B zk`{$iJ##Pns*>Nqs4l4bP3%*<83jh=f2!jVO2luuT@d#1U;$7J2UCx!c0KY`JcBbe z`Zx7&39C{v>YF55u`vS0Py6w($98#MN{JYKwK#(g+F;cNa6U(!FK!HS-`kpAiu7sQ z<}p$$1+IQURyX1qm_De&kYkZj;&lNfWHm&g##;p-GH!dY$Zn*|T4f)DnklZP{ba8u=y;Qk|iiu+9iUT}FYX_MC5)S`E`j7_~x!=NLbR)QAnIO7wcm5!JWTOg$ogY6feJVE? zvK;bhAy3A2$6jfx(!suSnsts^&etMXlc^Ro%_or+x{&>I*o*#g5LDbqixJ}3G6zE~ zp|%nB$DfEa&p(V@Zvoq((A--mj7@!@t`#yW8ew5?K;wHG-J}9af&^m%w}L2p=MX{>;{fx~RQi&<;n%0BJ_p+9}Y4Y#SfpwQUO5QWIf!|(m-&S zn@pqH#*nU5X~d8NJkxvO=#-&D@ZvyCvwzEe@e7HmO#ll^s%t91fJem2!Tpqz1?_A% zmdt4!m3vxcHlxf@#T{21Z((3z$5+Sb6c?PEl4*%4)-1I)hf{%Y6H6c0bFL3foxYZ< z%1vywG($;Xz;us5)Xz)A(&AHl?vgE|h$`U((s{X-{6O_>qZw=v^m-W?J311e9@Jw4 zG`4r;3P+gR_t%O}4c*xFP*Ns*qQ-j3DcYk14jCmje&q8TtzMyh+W|6<@q{1;cE=m z;C%buo*$AGM!oc`@`E&S;Pe}ysa^eEG!kSFd!sVZI>+~E?Cgh~KL~8JQ07H@jY)nR zaWFu_lGWr*loAA7Uq6vJTFMw+h`ed^H}S^9Ej&U=^`vcmLe5h>ne9EjX~<<7g;Ib$JCrK%9a|Jx&|jcs8ajnvxtHeLW!+1D1Eb0( z+eno53n}ZBKqYS&BAVvrCh4QmeEXFvfKR_Efa`=SW0+eN42t;%;PAiVw-A>?ezqIq zf)_loL*iBGQ-KVg+zu`JsG&51(tqUr(D_Z2{_FXx|IDn;QgUCLB0;A!63hi-30Ds9 z0NFaJaEP>oKi~dEE=u}-qp4~3aa5wjZ`B5@z>*3+^lmhEn2tnl>#Cyf&Mlo590%^WB4n2=b+hO`p`eKXnC!wQ>^Ip)@WNS{E)q-<sk{K?k=qC31kgvX9m>#X|x$DoQo$oY@#VKluaFV4{Le!I9ivw+o5gRb{5 zKX#ZYRgAAAD_BIrTMVW8G_yag&~jHinNh#V%9PpYx~WWVim_ z7MyNPH!iLhwbtB?RvBnKt$_0@_kkZWk~!6Pwo`=XO=mwZ-k&@f)`X~uISY_b4IZQH z{D(pgCEu6KNB9v&0PB%vCB{sgbxfkZ#*T*}k<4_9bkaY8d~hnZ-5GJ)218_8qUl+y zk>B#O{phKx2*b@B0lBFX`U{tGup=tDr^~DSqSLu9r_5tSt<_*c=MW@o+dnLKpuDhS z3kknuJe~qdlL#6Z<#fLJxFYbi7lXQyM@4f^T0a{x{g`@|M(ApKgQVJ#?OaHp*-wX_ za&=|GZNJeW-ap%Ief;`=b$6XnO=ayG25Hh{0D(Zn5oS=SN>Q4KN>L_+76J%F1?dox zYG@81Xy_5?5TyhNkN^@|C`um?LrDm|h=33R2!tZVFEeYtwdV6W>)!kCuDgHibKZUa z?Y+-hd%sV4oiZZZ4{)Yv?;{7zISo>ialUfi!;(H(J`c>&CAEoJOeU>J!GA-!na|Qu z`$t7;J+TSXY_G97K-fxTS5=HI7EupX(h~{yeSp1-#ro`K6e!0!=Jka)2kRJM!BO|5C3H3{X-XN9h6E@95~<|hH_0wXnnFbK<~@`H-PE;`E2V7Ss7F$7Q*SU%Of=s zAK=puF=Lu-?fvJDNMn5EChqaX~Ejm5}piI;j3g$5X| zA1O4yiW^VUe&j!dklqc2ZOB{U-h_ugW4hJ%{8VkHFt$mm@50XOMX*H+IQAd0zi;r7 zDry#0vmn4(t3xeO+~?1II*1xsYvvR~9Lgw5^$_yil&83}V6}p+E(mRk&u+OkZU+=u zYq>dOvdUMM?^*m0bMxKdNDjD(+Rd${?XN6VCDWUF*-XyuKR4>c!&y5!$Jxm?P;)t1 zrF+5Bocu#TRnYqd(ckDQI$XPw2`}Y`K^c@3N7oJuK>LIA4KVjm&$?qpWn3jBBS&q1 zF%0Iep!0ALy1cLOA(?#X7Wc0r!fUk3wwX}+=?F>96MDemcXun*D+AoxtxIj)$jQHO;S)uuYmgsG#U(TR zU{T)iU2J||&6&d-@A;t?Dd%uo?V+B~=ZhfaM+}ieJ4p`K&n!%bO$w6`iFMQxO zCf`U4i^1GKs=kk*SyI)JDxVz@n((&e+9f%KlokC0gEU|9iK3M*8n{*G^+M3+M+{;( zR6t^Aobr`L5i|^7sx;eqh&Tk2ZsA|118@!y7YbWY=T(dU0^pptTHs-g?k(T4wmp-+ zP0D>lLWqY81`Am+mI>&4e8E%q8OV+Yw#)m*yGuA>_I@zIyiH~uoYkC|Gf zh6`e`*2ZeE7Av}H=WE>^e3j;8DxJav4{vx}JiIz{i93IW#P902ldVPxo`%?%a*r%c zM;t!hzVn7{-HGwDhk<`4Bx}yS1EgtqHc{pwwReo5EN1l9I@U~+LCUGL;!FCL5B~kw z;!hUvA7Xjlm#vZ)I-ee8v9*kvPw&0#+xTPb{+~4Azci2EZreQmfX{BTBR&T0{=n!g z&4%%l$9&=t*H}?^bz6YO0$mC^z+~-F`D!Af;QOTX= zWgN)GYxeog5HU3rsu`8?mTdv^(i2|#qVCLzooHnuzNwvc>&1}$cmOSL(zk$u%e9xb zS&f-6vmAn$NlK&fQ3`3kGqP;DhZiHacv^L1uD2PVr2vr6z02x4p3bBohL?S*X^DOb zpBzFb0a}v%!H{aHl+|5D0Di^wsD!?02o1Mw@3}b5o?l3matKY{r29Pvlyrqu)yKEW+^>P*5} zC&%m7T&^;rlK0zD5%6Vu=c5BMjHcFm^ong6_V?gFzWgoVd>5ip`R>gwjleb%tkIDB zL64bci;{ww(=P2SM@?tEoM1a(C6B$cApG3l+pP84IIr zX6@6V$C9kQc7v>}Xj__o&*)7nF>BdpsAwV7WFM#aL9ycZH@j(El%(05R$FBa3EFP2 z0p33eWnxI})0K5!Sw3WPGW)3zeyJUOJ1t8dN=GzN5r5K%bmyvftTI(D}tXF+` zDrc?hFbyBJy#0KHpc)vB8=*GC;kO+M5(VJ5ty}_rc{VkbF_ik6gzqb|ZPf($7L0|& zmwpueU0P+80|3`pg3E(PuaFiRr^I`=ABA6nLv{x3Vkg{7TM_n`{7VCU9Y~Z3rcK%y zjq+J|U|=I5))&neSWRDSYNCv!4ELcDjeGzATVnn8+A!;_FntSe z@rFk+owfRayP2_j9H)EweWzoQqo39!*-n7v`LdpEcNflg+cZU^s_6j&rbbMj{cr5G z5~~o^X1NfpF0lEwp$s5n@Wv>x3REq4GOhZABx?^XTer*mor z!&H5>_taZ!a_o-z(iy_$mrnpJPWjD(O)Q0xatn!b+^k{mx2*Z)PG`G0%v zDJdKqgif6OgdI1AYLqV$DtZ=cnB`)A>SHbAm#IsQh7<+8>Ns_7x!mQ!0C?zx+vGW{ z(~v>OqkMFqU57@L6=+dhee9tI>fv~I5-`$-acD|80@RiJlzecmm`#12I?mVEc%33z zu4nI?V0awCG0FAO!hq)yfOBWkt+>4LW(nmB3T=8k)CW;( zR&!qCuD{B;<&RG*L`ky46~mr!ti3D|ODajgW08OlurZqD(fQIN{ZBHg_?-)`N1Os1 zUjJ5`e0xxf?8(bKUndhCG^%=0weoX!|3Xv2od|(zTuV>-R8-$@`TATC0vN~ocRjwH zHCmqPZZr~eMl#V9XC@UU**q{XpiJB`-?iFs3}GB2cJN|RfiRR{fO zXnDrYtHA9rEbiA)+?m|soFHkr5N?nCc=jq}k>%2;TY3D8>NsUkdKAi6Dza8qiAJrh zXWE=M!@TDMT`TQC2#-$9e4}kmZ1)AVAz{kHjHgr+A!-Lxi?OmdY{vxj&Q>%1ABpo| zG>L%F)qP;=)=!=0Z|R#boH8Wn9fITUopMk+nIoVxaRx2BK?jpoq!gKD$#gF%9exb9 zXi#|r9ODDS(;zUf7pYtY3lMWtb?08w0kUEJ8xBHOeqx9V(8MrOqBelmMltx2JoGT= z2D5aV=ia_HUOzq;msJT_{@|8SOc_f_N$MZ$0E2n`)^BunNt&su#kq>{xdK^O?yz8w zV>1`rvHlsLduP(0##GIAf4dIfk0Yk$go<94#78QHczYQ5p!8TdlveJW)_%a8qVU=k>u08n|~mx%17tdtJ*phq!rOEUFJUtb=Vl+)-(O$ z-f+A)SBej${f?2Bdg<{(UFhkEK|Nl_YrBYAHy!(5N*dkE!FYcOSG0G2-7>PSKvj6- zWQGk`_(X>gZ}jEff)NM$W6?Wz+54WiZRb=hd7m6Pw~T}5&5{SeZXqZp0-T?pncTP` z&Dxe`x)BhKcw1DX#~W<4d$nKFeAJoH)0^6{Q!)8X*a{;%{1=*j9AX1>qrAm2|E+wgZXHzj%83XCrqXmvOjj6!G5q_QD z6=jl!Qyc{8c0{#?O*~-=I~ag0c3<)QV^7lrs8xgpsmcdqK3ja8*IFHw;jSDxa^res1;oY!3VPzE)I?Y}eKA1~SlYW=hp zwvNZFA#j_13H!x#t6+L3PZRT|?japL>R0wumAeprwr6lw^7goJnP)a;jh~jZedT0z z)IAi!J&PbR7Gp)BotNv-xqcgZ6ADo&z&=TwpBd_*Hp{lp%q(b?OaNw1A%ff3(YIi3 z^RJitV4j0p)MrW+u03+kPgi$X?lc(QF%)u9&ahhwsZ4LRn@jIznBKQjnPn$F3G}ju zxDoBKX9>WuU~(#U$q|)_#0ECpim^x*OX5C7@4yGYk*4s8A5)oN-Zfi!p-4(n|N0OF zq#cj^$)iL9-S5;sGq8K}H0uQF2kvq$@XtR-j34WhPd)FNzK!;cWRc$duWlOuMnL{o zPrVyr@m*109OfUCl!w-DF+`LuzGv#Kmu}#7rWV?_V=nJc7=RmJ)_#d!3ZGDv!pMbV zuKpi;*IUzT^-}W2VamLkZ#`Qo5W42!A&d1i7r0ON#QQR9KD}`YM)EfD{^7s+|E&pp G9sL({E`{*` literal 0 HcmV?d00001 diff --git a/docs/resources/integrate_your_charm_with_postgresql.png b/docs/resources/integrate_your_charm_with_postgresql.png new file mode 100644 index 0000000000000000000000000000000000000000..c17393597f7cc1b4ee797826735b0db688fba8d8 GIT binary patch literal 19035 zcmbrl2UJr{*EZ~J!wMD<8(^V`QbTV+AoLOly-Nw9w**2U0Trc)fD|cGqJTuY(xjsz zD9s2cy|>U?C<*mH0q^@M-}>Hned}Kfa*{c7=FIGwx%S@I-V>^=rF{0()l-KK9XhM3 z0)ZVmbQll(hn@fm4t1Yl0SZoDLMa-dTzu>toZ*K!gcJ_GaexHu-H<2_AqWQuWaaM8 zXX{{v@U(J4^0~rMKoM}?#m&~i&H-+FPzNLc65#xDV>VQC5h9Hcb>U!dh~+?kELMB@uNW z7ZHS#(m@?>xF^!V&GqkB^NG_Mq;(2~afct2BHY{%&Tv|(kRmXwKm7-)IQ;3cf(T3= zDJujW!nX z(GqYG)6>>>)zfkkw6jx&>FB5m%6Y2Ed5fuYfaJa9p{fo#XkQaGh&4zHA?~A$07^xD zRnWdJC>2#z4v>ItW)q8#e`EcTH7cd6a@N(p3nipy~oL(u3K% zD+y`X>uG?{C`})@3uc@vg?jdZisIQ{uY@-KJMH%Sod5L1&kRqZOXFYdSjGDTrjU3!gT~`t6At&K% zC!y)-Bc@Lqr!T};1u5>05Vi$=33z(x>lz5U8@mb%x+;2V8R={4>NuI``8vtj%Ofx- z1Be{N3Z?}N6Jr3(v^?AisbeGzx7W2pqR_ftC<8GYL6k8{+|64ZrQj$e?xts@<=_sp z7qvGM)^zqpn;0s3xv06Q7 z3LsU0F3Cd`Rg^`2t<;oNAz}^+E--l=RY$b4n>|KCLr70h6=kgCi!zos_6F91yu7=s zf}jLi!o%CaTUf+X!wx7jadcA9(RGG_5Oy#jH!mf$u@DrasB5Q;a`*7Gc7f|@*}{!P zbP+m=T2>OSRu1A2U11ePh&M^zK>oF()HB=m%UO{#?O0zTzI z0ycIa32QqaEf18um4UjFt-Pv&vM^FWRZ~X?tt1LDLh2|9C~F$3sM>nl*tozP-O$b` zI{;0TbX45M9Z;%BRZksnD-jO`gr}aZzNP_iM_I*19_{F)0<2Vh9k`K?imsi52gVr~ zpRm0!QpwRsOarQhaJ6z2hoS^gisCMM)-Y=oVFQq;j~-0O(bm`njaC=40U3L^Iaq0W z>S)LrDC*jX={n2XB53i zm^<15CKb#Rmep#=@Z6%ooBigF4*hV~i|cR3e~r@n!lgNA{<4O&aYQAy236L=fM z)yN5W#>huQ3#MZS(NZ;ZN5YKU9iak-hC&dKGThZoO~l1l&(+%s;V6o9SE8+6j4s9# z<*TmjY-pmTEaoBXZ4ERbhS3++R@R2Oi@10zT0?ByjE#Je9>P{GR*Db>A3Y6sUk_gp zind&|4Q$1IUG&k`-bh<_kg=65L`YM@S6>fgEAC)upy+5|D1Z?1^j1O|xFHOXUcmNN zGSXEt5|i`sfT`*E8bF-96+jBMaAA9EgrSm&hocqJ-O(DPBCju~DJmqU2X*&E$U6WC zs-%cfMJRyu^u6Q&aFKVx7z?04P^hkuwYCOK06<9zRROe>fS0zXv8bndc4!Bfny8VOjs!{><)$a9VkBs&p{XUI zWemJP&(}!M#LH96-p&W1i&8=vS?e1J_=YY8skGZiYY=8>E3F63_(lzPeB^2{m_+gNG=DhN@6?o>1<*a+4)&xJZ6G3|tANt9GE_lZBUP02O?*VG1ra(P5V*0pfvU5Fkg<@Ls~An$ zsB4I$Y@LznI)J8g6i`D8YwM_Kxnq2kynR*lMKpyxU>Ic)Lk|x-L3aluV4!HgDu4in z0r2&2+hE^N0Dk{5frJ!`TCN)$IwW^U6(XnS(>p(K+Ds3=E4vi)*!u8~VfQPF$JiNJ z467^yo;_`U#mQ~n32Ln(Oy|ev3gpRho5sa=3x9lm#znsy^_I8!bdpiNlcVJESDZHo zM{nIad_;)6BTGsq2?)Jwbg?}S9_muRo=Q$erZ)E>emq*dek?oWIb!FiGQ<2=A-ys8 zAo^b1jb!Jzr_X}$GYP(l*Jb$TPiquR*J!nUc02=73S>E&W1PQ~*w&V*mN3Z5MGwB` z-?|=OV=Pl}%8$|hZv4^Yypb<$A7I2%_FmDP%?}jFM4T2+^;_6P6zbk6|7w^kx`ooC z&+_us8OU4XeBrN5J0n7RZ$o;mt`HTf0}#D=)`K&ZEc(-CPl*U!rwMW#M)22HgaZ?H zAyH3ALyhj9@1@GzUU0Lf=#Wx|fulpCORoWjed7u>taz!H+D|rXr5jmSFl)nSQ+V>` zV!}yld%HDI`1ZnrZsNo#ynh@#z|s(JY`zxTJFplk{VZ}JH?=qmNnixk7rSglgUTAF zueM$o^)jek{eGGOr%^l#Efy89k4j7ved0PHa~7A))E6I5SG)Gm?YTtd`~?^H=&2&x zAj@vfiQ-WmpX%y?0jWaQf)$344((yVG`YQO`+<5<5i+;;iw$SZhsw3j`Egi-ml|s> zBKZkoVfeLncj80ax1XgWHguKkqVD!=JnOf@)QXuL&x9*FvP$L_$9lUOD7l!h;g$K9 z$Q%)~6B3l?ApZr$O}R1L=3?_%XNlCvTWsz1E{D5d+fUw!rWg5`j=!??ZT#4^dbv=> zF#PzJf|aC7tblsJk4SAl{@Y-MmHTavToB!a0Z7vZ8XU_Jj z*&)H2F9hkDQA*17cQa_cRjLEwx<4c-`z6A-Y)yXkm1H?K^fghRMS$*-V}0h^zaR9#s*RDd+mG`Saj$smmFNl1HtprMh!94 zWZ|sU1Vc%OwC26BG@cJwY!BrnotuDv`y1!MnSwsNVT8SIiKn{Tn2^IA(<;*$*{eTL zBHYgdqU}GCP<#)SaB9U~Puf2`bn7u`oy}#f+g%YhTL)?Q#Fz)KroGc|`dpYac;o7h zARs!iHYsQ`JOBaiW3fd_S}r$6e3^wOg~_g^_o(+Iy$AX6`V zD|-JBM3+Zu9YvU5<$gSJHtKc>myQJ}D}tM4a!WBImCVC=s;A!6@p^$?2;2SDk3HMi zP+X5x(xx<<>|{~TS;@B-aZTypZzoqP&&e>$zTh=2!kGHBDwkdikNhBGI6xVv%b8-B zh`k)?YqxY|^oR(`iay6t$ne{5PIyR~H%&gPW%q@V5)R%MF{HcTiDmd08d9};cE@$F zi1j3sa&eZ)G5qqV>E4k|_*IrL^<|-u;{+YMLuxa?b&6+SmNs2m zi4GckENx#d_aT3W zL;c@B+xty zuTjs+H1f%Kt}(u}HJ0s}cl;GxB~4LpaE~^N`_u^?3WJdNX!02MjW^kb82bwc4}6-l z7FzBeGG*!vg$@?+${-zLgi|{*U$&30(KZy}nZ}kh|MKzXWlh=j*1n7;jzOy$ONis< z5odCn{9t=IbED(0PU0j<-L~?Y2!cux#HTRfSMID7ow3eS*~I$|=G)36XbL+mo$UE_gz@BE0_ch>vcCl9~Se(|heT`tyjE-Gng17|TkKf+A*<64r8CgOWPinVtx z#IGn#X(x=Oiq;EGfutyK*}%;`Ug+2>(s)SJgC76cX)xg{{)NJtB=L^;2X+P z+hNr6IkPeAsO6MPSp@Sp*&!PD@R^3arRGLIW*NaV(?UhbT~SyVp=Y$uU#gXS+zIa= zuzq^ijGMuN@#lI$EXzu<8);y4XKU>w*rn^mOQ^+SedmDej0ApgnZbzpK8jisIorUs zXe{?+ak$lgufMwhVI+GOVe7mZjtM=u|Z4XzjQ@-bgx?ii(|;tpAHn1?d7NH zIvpi`D@VsvHVMx!=&3jR*K`v_pVGp>+N`V-QCJQb1}f`~eP5ieBDNsXA0Ouu^_9WY z&pbOl<6RCTt6&?EM4P{1VkZAzck|q*`&gJ zcx0_haGh{H5?3|)*{zL!$rWSSD2fIz!}~RbyYn#y%re=va__9vb7nh)6(YjWIh0Z} zx1@kP2IEpY@!v7f!O$J=wM0I=?QV*b8Ky!xvSj=OI7fO%f6h2%E7z5AF?a3eBwEtU z5jUG? z<;?{{2E%M`r-rbi&0?qE`MDS_I(_!p_m^YUWD(IW7qk^tfVvSAM$atFmCC<;Wey6O z>d0rwzlMjE{&=*WE0cP~(>FZ1{5EG%NZ&x1kL0;FC*H?3h*^Tyg%_E9w@74eucEIz z%xT{la)sQ#V%a^m-Z7exh8$ZoF9ZmURmSmhS&K3kWR0^*#1J>9-BT#BMP3>;T+=zM z($z{o`gJ;?8@D7_RG#k!tE?JF9W6$EFG249An7jHt@`P8gSt461oLyH!}1657UJ4! ztJXVAgd7sTwBfgCzi|k7mQd$48iSbCFqab=8l4vLoQ$X^B-7hY6x}+h@)2)-Ts#Fj z*xDLZ?3LkiT&o_%RMOkh@fQ`92lMpeDOdh6R+K7f#s$Wd{$LJl*-hN<(g#Xx$D>@V z0sCJL-Sty;1af1g@vw!kf6RUhI-YDF1&s|lO2U0K)fb}3lw1ORS12?!Pd9th^W5_4 z)!ii0DZHoTiVO_c3Wjxci(l@6C`~${U+ur#HWK@r{pDmasiKS!)u3tYvNCFJkiyIQE z&9#YUMLWs=DnuMk$(We;&+75l>X7rJf6~!fIPsXhx-K=0erPUisJLMH$QRJz6f$JA8tabL!=^&C& z%+nkGpHXk>BG7XMA}o@qbKPvq#&;6;{c)mOV^QreoSpC+_rl-gTBAHp6 zCMlmJQ(g~-(@Rz1N*~fYp_&m$w!GJCu0ixtHBgVgTx^@U+D;k1c{yBhoyGgH6J6;; z(V85nnB2Kn>2*JbV|ZVOyHMGmA(_k!U)O1Q6lHCewDICcecOTB^!OaVTt_T;%*^0* z-H(wX=h8RJRGuXJ`@xGPM8EBa7r$MmD0Ml|&AlH!)a)#SKbzt`3=CcJEN!6w`V0N1 zY@?SHY{eV6tHzkJT3K1fw9L#v_&IuU)KJ2JSX5Q|waSNXqvx{Z{6m)fM?`ex?#EU4 zsUcSfOxFCzt>@eIAFe-BVX8GKt{N>NQ=A!7yg$(!4k0XZQ0r?sTz59tc3lv$9fJ!C zEa;!hONM^ae$wM2OS8K!!a~8=-vP;ZuH! z^7p;l4G)(07~p$_-G=3vzdTEK$UjF1dRIB9DAJcs_66##zv**f{j9ruUBJX1cEB+}kW^OZ1W{d@%mq zy=GmOKE&8m@e%K`O>h8mPu1*WpSt0XHr0KbCU>Huuehp6vl*jVKJnqaT^g6z#fcfR zN$j~bPiK61dIx>?$kqN@{>2r`x;OOqFwM~HmFLLnh0c|NZbJ)QN9s-4Wm(_B-W}L( zNEbIYO0Xm=aHv-78BKyt!yQ$etGjfApyjxZu)NBEU&!6XkK4SpN9J73plOMDAMP>FfQ7bXrd@*UHKhSjy{`LbdjH5^awh{H6Wh%EW~6KK?AJmZVR_ z0_4Ke7gzh_yWJ1Ao^{1fE6Nin<*VMCj-}9P`QVX@z{aFNfGePlbe{t6$UMyw1O+zc z$VK1^)YV`D7BR5#F4NT2pF(2q*RGkP-m6E~78->0!}B}#PEwiArlVgic!RfEam9~2 zD~-oY)UTQo%&WG|t4+$KMtZBh=-YFY@wu6SubC6NiW#M!On&&`Ge69ELY~~*U7s*oBZufS-s><7Vrevn@Sx zhat#R2QW2%-YK|r@5*w|uD)zw_NVruvSTGK1^o%Hww{|Ch~OtO0iL0xXH>q^G*(C= zmRP(rvsJv)A3T4kDewbv>N5B?EN7&sxDeV|)FPyR?TN{z#x)I;y}+^Bw6=!Aw%Hk> zoSf9oqdu$6l=g23{l+u3?==m;M4F$TPBJ)Oy~%8H?Q-#VjcnVYdsUB%uhm0m$H-I1 zCbT_s{imatDIYqiAFm>j#jl0Ff0Jb0b>v^x*yty4XS(#oKhMB))!J0S3_$uFE;_>V z^9zF>%B~NZmsjq+DtsT=HP+=rnxNM<$e4Un*W`~GtEwQteg*x^t17cTM7?2rysrAY zxOuvGBQN%n?T-R4xEquKH`-Xk63yS5+A)6Qy*)}vl0`7PqV+x6C$ z>rPexQhIs%&Yo)Pk)%fN6ZLX0&|Moo-<^iLcKUcoc@jTU+tyl!Z?lVK&VcDGrMj%T zM7%ty7v%8M-i4?J)G}LRv(2-BPS!$7vT%ubb^XNn2&AI;4-{o1PE);WOtBqoYP*-~ z7~Zbxq&UV!dadoQetC55>zXdOlLl?Sn>Q1G?x<-kp3Mm5+lve<1oN7lZz{;7WDH%F2op;{Ke(Ul5L{cBEWL)PBh0M4ORZh9ZU*MhR^;-6VxQkQenZ;D z5|8*De72*3oHY#J+R+-g{N~M*_sQF-_|REGKi+wSA6~gw`zfr_-#THVc)mCcNoXn2 zCngY8W7HxsZ~N%qh@V(l@EswK{kXRSA^FWXBC$%0I z(^PnRELOt4dP;Wz>fLb-lD_vNESRt5W?_oA$3kp5-}}|))YS*-X%WIcP^A0C zmUIfvT&+P)oH@1lsZ=p~=Hs(jZOli?G-MV*R^Y}ZD>i+I-YN@kynt)^COP)$&yJk%I`#eY0;M~F%r>u1vaw8%svzp+B9 zs-3Q9UY5T)(X5wUfBDmN^1EX!V3anCBpgI1w zw9(FGDr)FIHgn#7q-KEr@UMyeDZgLc!RUwIt)^=}bv*KIcfByudfC_|yB;xH9m}5lx@ho~XYYZQ&BLw?jLVG# z7tW+(bN3tqcv3_{yVss!d3s2rev3$($o}n`#XkClLaQ$RqY%8_#1)dSj!D2Muy=Q{ z$W?3FSUCrLZJx~>A6jgS5AQRH|By9OANN-*U8AEwf7l?azhEu;F%FlyENK8HpDbKX z%$6|+H6=AvD_zO!f_*itlI{6&YYglaVdlKDjlftxJlMKkkKQ6$NJYf>6ldIu0e)rS z623}g74qG$37FOX0RrYt?Kh1{q^ff5Tf5$ldbCMK&5bYUCA->yg|7T6@MpsO)ts#j zP5+cjT8j$5Ft_@IN>e5=fiIyo*Vq`T+6K7sZ@$Lq&#F{(c8^nyqM_cIegS*CYJH`r zn_cKW`V^wOO4?SfXBXJklJRWPakZ~Nd0`u0lhgpbwmqXiyMFgug>=(BQ~?k{{Q&<1 z*6)^dp!mcI@0x(2tD`&JZl&O9mc55wf1&=k?VE8!f0fJj+=Om<59Tqt(fsQ;=S=ft zDL8F4+knZe8}xRD%O5GR=#P}~yf?&G7COM}W;V(0odKKXy}4Y~kzXq-`6_IPO=M+= zEz$0~$MORBo<3YmURj@`_T#{X>6JL@?-dLxp-*WmvCKRsyQk$@H}~SkqcIBB)V0lj zcBSv7{!M>`$*kv{>{9>C{&x*GWxI9g!OCKW0U(*1F21#viVx*q~R`=#$G_vgmwl^Sd zeu5lv<(}76%v7*DUG1*(FnC_Jd3~ka^Dn5Im6cgPr+Z%X>aCAvS{~H3n+4~gzZ(d5 ze$C&eBV(%YlAthzP$T#rVMO>cejN{&+KlZmHOgf(d%Yq4CB`Bnj?5VUhJ4miG%E$l z`RlX)uEW#|v+XZ;Qd`-#a+8BPUSNAMar0BUT5eY7clpz*Nj~jizdR-CIlA%^?xdB1 zS>{vO_jy;MT{Lzu`d<&Ixt+J9+q>gGp~??yo?v9Irby*vooz*o+O%7GipGqSEN!|3?7?o1wRcj)Uj0XXZ)`w3@IY+q?^5Hp?A0o~ zByPUMIuqHl*Kep93Nre2AdF}XHb76}Ea7H#Z67e$iH_Yz#9Fb~n$LgRw*R{!_CMDo zkT)5xi_I-|M5W!_Xw_lmo=V=cKmR*Sr!rap9L;{dHGC0#ekt@JVEcaBZ|eU?0de!v zYKi0{78$wfYZp)8Bg(#rY_y#+S+q-#rJz*s_spIeJf8JzVZ%yJGNmQmIMFJITbh1- zVCE)(DX3pk&SE_HrRh&V(R=I{!q_I@Sbm!OOu*qpTewKKu0ALF&WQZoYrqOW53K4F zG&BAQ^yq;za2T-6|JwI6X+r|;&_+cY(n!Ywn1H>rA7hhXwX6KFvH?8&W4NgQD1OO- z3*YkwS~3$We-O+1W7QS*@wFZ2Qf2L;xq_zHxcyKQrCc9Uu=~_amt)J~5tLFK4;>-G zD`oO0v*|B0KhNX_Pm3x2Xu7luE8W^X-7>cD`t(zU!4DLT6U~i_RO1!*Tjp1XGJQ+W zEiNuOh~YKe&V}z%ZY%Lss*c1eLS&0yHt+tB*7Aw5SLY8SemUH9y}0IQ`cy7H)J>PA zZjB4uOWfz^y%Uwbjkd?uo;OEMz*}s5=nSqpEMvT$2b8(WoE<9n3@X%Iy9S7mrHgng z?HxmxCVWGJ&Y?;Dxz$5}$Cd_{C*Co1Tb6J!;nuW(I%P8KD)nSGJq3m{wnjQU29BIN zAcpOqEfU3)`F~uxbY&w8neZ)dfPW&=XJ8cB?-bkS%bT*+8GEk}kna3+$0E`HGydZI zDSbuT_E-n=Dz3p6dQ9&}vTIwq#6D)%liGf+j}5wz?|G%FG>HO}Uy! zbC|3pGMIX^zO-^nxKRXQrgy3-GFuI3da-`fxTq0eoZIrUas9=Zua^jy2~NW589V7foU--VFKJYQJuY~? zpg285#`)2Jf@h*0zu5Z;pVY(pG=Jbc@Xj@??;gr#94pxKPTD7MIffn{Di7OvOIPb(>jIqTZ*ucEhIqK=NAwu&@{Nm#T{;OF!*e1BAOsW40?6>AHuMPa_KN;)U>*7rnawZfqt_O_&WlQsfK9 z5&N&pD5H!9J<>7imF*BKf&-PUMTqMX{&!{*m^mgnfqe_>N2%W)JlRiB{WzGAR4I@l}lJ(XmxQ> zip>>ZRR4Lk=nKF!JdO0-Y~dk+xozH=e2lj*7kyA3M9;7`v;C9SWHUgo>%M-*<%c#u zjbAdB!Lu3M)xb`cdM&0V=)KOW1Qu(NBseEJL2vj7aK8)8fX~rOpzgRh8gK|^%^e07 zg!xu(DQ@`yUGs=6_QYt{OM6 zlB1`(`jzk5S0`ER4tLzd<-0SP9qpQ+0dvbe2#xu(*&ObW!!?wj%D*Q7!16nEx2q9= znvZ~@2h=CfZ)?V)Z4Y|_pK&8z<6FBhQSBh848JoZ!H_?M+zpOd=pYH zI4ibg`Q94Ijug>jFJ01}e7(lF?Ush-*}if!Df{z1LMum7}M~ zmszI2mj?NN`v@txU5BHtwz1b7*^kx&D!20yF~?@R&g;N=19l|GK*u=mB&Rt_()@YN zMV+9CQ~hBLZqe9XCGNWJ$>11D+*H7eC}WL;ufX8ow`2i1RV?jew;i~a-DF2&ecHGN&64fyw>%b0sUki!V}0`Lcbz=CjC)D> z&?Se;AVxg%qB;!Q9(kl{ll6@$wn^#^IDOk=j~-vL)_B_pNQ=S-%@n~;P;ws=Q-m@o zhk%u(gU#$eS8L}Vxpwi<_Bq8XFz8d@ri_Udu6R=LVVW;fhm zh_(4ck8c;@SLRcIx$5jE#WL?yr`A}#4Erh+?{oaZy`j;uqM3aKt?20S%%ei72+}L- zs4&%;tJ10v9NLMt-&2!%)?Ih0TZTPebxr+OL>g@(+(TMtrIKWz>R`WJ`GZ^tdg^qTk|QJDGtn$E4_}^rq~Tb) z!}JR$(l*fV98!!q+wRZLcbP*Ge9LFp4fr;BR_{p8BK{}bCf^Q{s#JaBDl5gfuUB@% zr(Wgg^F=3@qaGq|eDG}J+Sl&=ql`%B7rqgwfh#59Fw^&r6)9~epTC|H@?_wTnH9u0 z3y*HZ3nxO7_WMERNE_QVC|8k2h9Yi+*4i-R@)N<7YYRfNptr#;F<4HW%_DP*=}(1> zUlJ3E1)-92BQza?9MCJh#g;*f{p>f>vC-1nhRppVm}o2?r{ruNHwiqn?;IZ9aA?mH z%Y5gGeJf+WqI@|OzbmRC%_V;|i=ceo8Ipq6mq6tf{!2!^Oxe%AXN)!uns))4 z$$&@-etBx~MY*sKX&}PjA9!vCtcQh}4mv>5cYh53cpUpy^FPu6#t3@?5We+XlrZPLY&JLcC{9CaL*6^z@HU$5Wl=S;R>uw0 zDg}sIfaS06C4tk|GCey6ytPi(#t>~JfP2LSImF!j&FHL}yauM)Hju#l_KA`WP?KvV zb;0YVvZ< zkE5h==px3}=Q9^dPFQiys(F+(x|#3n#u=3#v}CS-3*o4(VqVKceHOCOJGD>W(qTpI z@Lj(eC)AJP4iv=KHDvp*(bY@wMNeyE=WF)lGy59buXJS1as^?a9S1z>1$6YRJHCCB zn&DyD0s9vq;CWjPGPnX*q62CIM`Plv1*%b%wPpSb^cC9T<1AMthnC)JtjENLIUs#W$3Y!u^%tDpgnzSzTab0F<-4nIp#AF5Swn8OAG*S*Hx!G$g&S?f z)KM)@7AL@z%0j~NTSsap7H6_;R@>aY^L}OsILqoJ1Fz)`m*%alh0@%I0<(|S-Z;Rq zgwkTl7jSm8l%_!1EB!BQtvuYFxxD9|GER0M`lMV9Dy%kM?(bfq&TpRad!{Abp5~Ta zuF9OE#e92#9mswVjmPa~ngv!cC)x!5LS`^r$?;EzUoDMXh~|vBqqBacJUD(rbM2I` zqJJJ>zQfEOWKWKU+fU5hd+sYs`YHN;__~A+o}W zAA>Dh7E?aq?5J3`kSj`I!REC;`Cc{D*l)@MHZ3bA>(0Oq)+FE$6L%2r!tU(?99J4K zCR=*UjGyBCYh%%`f}$1ni|wq7?81O(ZOsJwL5wBgX~&jru=IQ%MvMtUp&y*vAi6wa zwHj*Q&m~dW`VKHJ`PuJ8m7cAL8Ex_W4ULuZ`sWk%gyqsrFKY$|`+E7?Gw@6G^ri1e zHZia|MYs?!gZKA|hWqPO(BTip13(A>g`pc@ zDM{uNCHHMPi1;gLhYE+fjry&C^Riof%31F)8{({)RadEgC$)}UozPdj$^>ZZFq?r` z!w=2bGev2f0}`5z+C2pIiKB$Rtz4CKz)@qryD%kcqIGu@h{nA1@JRYp!YF$)yLF1k z;--fgKk+l-J7Y)byts)Qpgc48d$DwsYDJn@`M@ONW%NNKQr z!=(0hF&G?aq3w8(6gnY%7eK5_%Dxw&cdc_q|2^kWAG>eor9D|N^z?Nb?$OAd%#?e| zC2gJC9y=b(UXLJJ+S6hc0e_LRY^6hGKgs5RofAY;N0WB7SH4}gz|;<9UArx;ClP)M zJfgea81pQlVkzMKJS|(#K7550-yL5aoI{M z750zPZy`i4v*+P;Lp+lbM-fKU^{no`wMcGY zEvMp!NHnSb$-&z1BoA2gAc1Q#1|eM@3&JC+B3=@)kDalZj?nn$E169WdVmrrgr@2h zTUHi2=a;tLEnK@Ik-hd11*&c@o&Vw)U6g2~Ip&NHAsHCG_ei2@{WM5F=<94CR!nT@>KvOkuG7NYTH z8APUh`Fo$eBqjmnPd?SAdG5(1jXBfCd*nZ!J*5FunPDltM$`Wf@E&|L2hIZ*rz!kk zeYnF42NUN69_u1Jw7l^dAlDdA)Q&uX2Cs4V2cAB06x+G-{m>~q{giH*^2Y~&r9R>D z7bBRZh2XyEuk!FAv6q$UKHv^wPi?GxI||(15uBZpUR%fa+_wr`=I&mGtW6vRUwaPy zoQi9y#<73QQq0F$^KMv$q&24>@ z`w#sgnf<8cU-+}lga?a7`V^E0vXaP!=d5bXzv*YZC9RavqdA_^?8tEzZul4P8j$&0 zS+j}2KrQ~H591De$WhMSyPNQ|Z!vcK9BP5Y;}g2?x2AKuh{>`tS|cogE0{5gJVvHo zBV!W3LszfJhT<^y@f7i`9I4vGQTRcyk6IVGN$~O*$6nMu4fCQi=%mroVtTC@h;Pe>PhVQgQyi(0Miab z8|k_?srWSb+6(BLxRPD3fJX{BWo+p?r0R?%te61*gTyl6j0#a6ky z9q)jVok%s~`|pfTCb4D1LZaUMbhOUkQqKddn({N((e_^m1Bv;$toCZ#TcEEf^3+A5 ztETT_+18cxRO_&jMcIPJ2GSbmp2)Y-4r#a7AGhhp6}Nme-;MuC)P!PfRc7CiVimr_ zkDadx6x-TC)_PC;%`f(Sv26SkvsvoV#x~JCvrwH}#C5&(+v$6;)h!KTppJ!(2Fhd_ zZ&i`Gi~4KdGB@b#NAFP%Wr+KQ0|D0!EMc+u|);ZOD#`enqufngq=^6LX3Y zGqs)IMeL7{by_~VY6qU6MS?}eKrLTVB{GPS?GxhmYSwk7RScwZ@^N2=-ZE&VM?7n_ zhe;H?=+nO93U+IkAX=xrFjv0~m3ncJf^x(&DTjJ{I86$cKL?Reu2ce9*fvbB!nPuh zD3$6($c}}XP8rR$B+W1ZNu6I1DXb9RfcUX^(qV@$rYoCQ3>!%)T1gFqZ5?dq-1JXD znG)AA7J!DyZMJd8$3WuW@)@(uK8NHGeOdD7$n+#8+$+>VFsOzkTjRf>N#3bPnw7FY-T)v<2Ya;NwY92OhgQ8r#4E#$%pabUkI(| zV>dQpck{3fsc@&Mdlg#>7N>SoeJd-<%Vi8_g$xmd;y&*hYox#keGdaINr(2 zYo>C(=tu4EgrpUyHWKi`nZxK&$iOA*7XW)ESDQCFJODj@nYLACSfw~B(ldcM2ltosP;()ix z!k8#$*dFkJ^nS=1plFPAT9)}nTl;Pl1#prj;NfM7hC#&R0A`X1wdy-gh&!Z2<87{Y zh0rkuy~2vqR14eLnWtlt%k-*i6E{TPaTnA;P_|M9XYglAG(X^i^y#OpLr7b~xtLsnhl#mwzsqHes@8(&T z2@-KcRS=$)ImC{SRFd;MC{kv@?E|g$@k$`~Jp=?KSCF_*n;CoHWk#V$9(;`1p9}%f z0`qDm`=gIe@&c@Au6*~lT@rOJml7)rb?|QHaQvuTb?pOIE1DKzhZGHsm*l0B2iR_O z#qM|{odR@z;9AU1am0dqBC!CTm##!fD`HRF=CqPKm0L3e*u?GQh> zvfXyGbjB=Pz>0>UbB^loJK!U!_>AI?zT-OQc>Rpu&iE%>zIw`Z;%ZuIH>M zd|5chx74$=IOKM*nZEtl-WN${`J73&q0;>1IQG4-*#}9+-L*%SZ(4EDWpA?R`^PcK zr03bKJ@(#1M=ai7x&2k?wB)ibjHLXoI-VO|N<0<_MXXUP{ropn&^dVbPPxEV_8eHGye2( zyw+@yMQVsydFlS+B@)r45sU>${clq$%B3Q!QZ*dDELRqq36Y%3EV0TVjbNnOb7dJJ zpTv3O=iv?vu5%3$@2H(Z;8?~RF_kfg53y6X!(l-mS{b;OdE54G+%8HG2wi+FEw8bT zDRUc6x42T(vR?jRY5FUe$!HsKjkc*uc#+f1&#_e zSylxM9c0X_CAQKAT8`^o;OF3f>x7)HKQ?Z`E;FLO8T)EAs|4M3vAV1*j?}OiIb760 zH{7URptU9VE`%D3U#jnf?q-Rk&3O;~Uik&sfg<3;P>Z_@mpA#BKdP-$Xq!!CWh6)>l=KR72Bz60Yz%)6%{hJ~;K&=``V6=FirqzLAdC9jb!S+j!z?d*B`F7eCKPSuu0+l7+*aK7nNqF`Elgr;A|A3~q<&LO zH;5`BK7u&e-OKW_-TE#t9WbZIdL^zm<_+HTbXP+3w<9kwwJM@hwvLqd;A921Gb<6|JS8T zBWl^ZIo*m%#M6E*XkO0hcY#L2%J=t4z5HsV@cow+-K12Z_x37BW4xbc-vR-`S$8sA zi40Q%PmD_#e8;W)k}Sm8Cd}z#Y`fp2W?Lm5yz2xWEs;EI-|9sne<8_8r``u9e@%J` zFs+XP9uVR7DIl~xedq{3V4MSYyd~_+C*J%ryl+*vj7bW#_6Mx_!Lp-EfR9o0{=z4K zI;4KFJOp?S@N_{=m^Zta}O{L5i^}+SfKSEl^(NQ0lgVs+b}2@0ELBX^50n8n)wfiuB^a|1`^>?5WF zp>%6kNO#NfKdpWyqz_tES^+UKy7eoh$21PM3(*5PV#ffUpS`#@!SUq{u2Z+*zdXwS zOK9x>Ok@yQFHcMN9p+*>76$Fl?0hqrVae`atzhJR)%dBwdWD_BvH_f<1 zRH`-eX7}C}yD!hOd?}1MO|up^lM5g2l&s0@ld7SEfIp=aJ2ZD(=7JnYE8hYwo0X1b zI^|lo=XTT}35k=k7g;-w zG{SX{wUK^rb)2SiJ%@g^pP~x)*onotE=C0ml8b$KsONn5kCh%Arva9ke(kQtR!jl$ zp4hwnPIxUdcbXv!>bs2Gh!NnJ|uob!Rns(sPWNZ8RR8|Jd z?GRTrFn4)mUjmN@p$>P1Z7qALr{8`*ZVrCP!zMozdPhO|l}1`Ct&{6l+!J<6xI(b} z7g8!e;Rf}f1LgA$wU74`44v?M=Wzan+miRFRR;-)3ydot5+B>*;V26MJM+xd_b$1o zWS`U&jTc>$+QtIP;O-VSYmd9*rH7EirI~?-Z;^rUL5CI|welwtC?0LJ?D*02|o3HARD|!(A z20<&b(|RwTW?NfLH0Cpi0vw4;w4*xvvhBkEe%`KH9(&!2L<^f|YJpiVh=OS8 zIC8kbFres9fWg=jJn(FlwuV@KlfX76;sE({n`%bO*iO!Geap=JVyoa0?>8Xo2PCfj zNnk$Ey!~O0<^Guo_YZ+*+euHvLQ7ZEMkC(rrS)!N=fS*3KLN}C%;M{g>06<@me+K7 zaW&pUJFIwxL)gwN;{L$~Zkm#H8Y_yOW_&{bw&b(|Z+dPVU~UL>f;39bkj?9+v99y} z+yP%_g{iF_HMODOt*%^nq!{zj&%aAB^J&JsbtQ2lux5LcW_wt3*5=ZMSKk>OfdYk3*jI-bvjz)f5Efo%cxrRn$JP?#Wa z{q#>_)88Sb{{zHmbk%=?*kSPbv)>T7W1Mtydu*|4|KFBXWEGu=2Avw^>FVdQ&MBb@ E0J%WL)Bpeg literal 0 HcmV?d00001 diff --git a/docs/resources/observe_your_charm_with_cos_lite.png b/docs/resources/observe_your_charm_with_cos_lite.png new file mode 100644 index 0000000000000000000000000000000000000000..3c5a58c45a7edd6f3ebbc9786a851918c11a2fb1 GIT binary patch literal 45611 zcmV)0K+eC3P)Px&08mU+MHm|&78x8B7#kQIAP^B27aATbDl-uh7aSWR zA|fIh9U>ebBN`nc6c-yB9wHkWArclD8y+JpD>N4v9u5!`5fm5~7#tlOBN-bYA0HnU z7ab%cDLHDBql6cTxA;{B2-pi6BimcIX^x?MoUdoVqt`79ZOABUtwt?B`O>rBorDVFETe67#~SXQa?dRS6E*a791KGA5l|VU}9@lSz!zf z5-~G5b9H}jaC&29ZXqBbZ$1|~JwtzmCmAAs{FW6dpJ>H)?8gGBi3I92^!F7equ*IXgmy zhmucEQFnKFG%+z585ufQI}{TWUR+@W1qmZ6H-vagv*;fN>tIrJZ?gV*LF5fDsi;bw`tlfQXMSLRWe7fpfrHEWmb9EISpT zb3WOf9#|kIKwt)DnOxgsH2jYjom4M&o!9Q@=)DdOp?esYBQWr1C$(~TQ-n|d*$VSS zLTIaCb;E1>4h&g>&vy|wVgLXj07*naRCwC#oBvM}X&%SP{Xj}`Nt0smhO$4byN7q6 zU4L=XuR%p64^A0;$@Y zLeZjhK3i0X9FX2m}B{2m~cSnjl;t1vUf$*Z{qSwAE24WCuAzA_b-q zf}&IdAQe+cEJ@;Io%%&V0{kuQS!AjEq7p;^S%0?UE=<_?o>St! zHP=JMwwjm$SR%IH(TmFzg-`^wnw$Za2tXVF_N}PIsVZRLiljRpC=_UdLh8ULvCoq{ zQ3oNU9*9aTYqJ%*dp|V;n}Ie}#J)&+pK60#j0kC06;d^GUy2d%cm#kF&C1rqZ9GT; zrirNQl>q7h-}5xxmp*JeTGm@KkgcGE1t&hDmsZM{zv3vJ!-tLY6~|_;MJ) zyho9pSr-qD+#f$=YNWe;#WeVjT6Ei`>%12u(q!FZf}XP9!^;z z0s+^3;iwd3QpbgKM9#QnCHP!ScX65fVw{(gkrcF<`Ipith51yY5Z{uM^Rg0N7vooi z$QyijoRnKHm7U&t{Am@4FRVU3Eaq8|Q~WaT4VD#%K{+W^@Bqb!|Z z6G6|4kQ@_Y5owlHLJS=zcAb-rLca<;D_*Bk8m4`82v_+9F4itAlHtIfMfSOzS>&Yi z;;iuG9}+7bl!YW4>sS>eCB>#$hZ0G$bmBX)MPXy>32q_9CWR+|XA?8?N{6_@%W;5g zQs_;+NC~sz3MWP~tEp$nt<=UvE;F;5`az2E@}d~uS`gY3&l19{u*eBaay=1Av#F=` zT=LLX=3at}&2yqBbCZ=9Qd?=elsMtwfHKqra2g7e*-Ecxo+kjZI4^H@Y)Q9SpK?hS zGLbZBDEZ>oja`ZC?DDcVhiuR{ONVpF;y!kn#P=++uJy#-h$!5V5^UmzxGb%R8y#Hi z(0XQgKp7Th@yG`0KYvLqgwo!xe< zfZyeXS?E}^{0@+J_VC02xfG@@dc#BB=)t)*^6NG-h!Wo{5tne+?}N7E_#eZU$hPQ} zaQ&REEpP@8*W-PH@)S&F5tDb$MjQ{-d&!}taI`T5T8Vz{1?H|cZ{TvKmbS_G?k>C}k4thz@&{x5~hE2iUH+I>XL1Fm?CJA9OH_P4I%P$yh z0!u5f?g6WFq>4ZYw91C4fhz*6%~2RbfO`nkrdkm7p@4cD29;14VxS0{Y5_wIAOk=H zZ4udURNe+!6l`q8HdQu?$k&8saLreAa_VV-RUg=a*wjj(5F}7UT?ndGh+Q}?Xq3#y zwSa3SvvDmo?g!%s(BbY?WIq^jLhGpi4YO!nA>Ow$Q&p#?E08)(nm!^yI^Uatn+ka6 z(WL3qQH9xBTVmB%^_n!5O%*j$218{jWvJBab(%DlOBGhkU1IfgS8M&MseH)l8cHqq z;To-9HI)z9b1G)oBdmnT)@uE#sm#be+I%|)HhzB=TnvLrzaQqauZCRte+U@m(^vym z?{w&X+F3n6}ss`N+r~H@Q1!O1hG-;DsO{F5%S#7o)$az_8?ak&t-@e3d@e_Sx z^fjiRW85r@v7Ib4I>hm(mN+V1EbVrXJ2BTDDU%h#`36Sk#*-?jJXIq(`c9w1Dc$%K> zVi=aOGp#(|&d?5fZ={{!dl=t19dXzvFy_3U?do@2_nka@dC-jwawGm6vbBHliTv#; zjaf~l@v^lew9iwSQIn=Js={d~k192N{4#ussyI*{ zxE|~FYx>xwd;BS0XXQcGV1g?!eRXafrZ;Q9Hu%Wm^6Ge4P1~dIEI9&B=Z=iC49LE} zO{dZLp+_{BK#y1^yYghgu7s)yaGx2mag zX5Bff&T3UHbZSDam;aX?{q}|Nn=8Zm#m;Qr+?v}K?DOsF3hnBv&H0P&H>3UO$@=Aq z&nCajjsx%}@X4fp8(GuPe*>)(u7Zq(>`>9JsmjB~{4Uc_UenjV~lU1c#_EtUJt z`bhwb(`fE~Tn&o!7Q@tq?r5#9_JUsDJ!NqQELNR!fAxN2t=?!h8+GO>yvEpM=?(zG zrgYWjf3t7CA#J329Jfby4Ey4SbBbAJ2re6YLT%Pe2#LDbcF{N&)2hKbaq^^^?Ji}v zM>#Q?%yUsN#yM_QC(~(2(ngITt>}xT#n>0y7bS06u5XG5w|B7RUL-7(d*Qo^6bj{L z^2ca<%dXxQDx2?{%rnnC&pafb`8~hq`%Fjzm7&uBR3o#jk-GwNS+3Jl7BwhvD4&u) z0ZZtV!xH@ffoP$ECYosFB+#%*yFzoF2F2J2C_X+gTj-6i(Q^W1~c60>CgVl*^K z6)S(w<+r!Du`B`kXgSc{5QOaWmQB^GT;8o^lB7*J7?~qSor>w9`BjvGJj-2Hs}=vebHU zArVfg(rv#W|98pNnYFHP_LP93q<>4@t$U+T%0^$RN%U}p)X#$``T>vEk4zq}9 z-9-REs5J!Y{t|UYwK&dxp1jQnS(HSfdSj830VM!UKz@P7a9nMQ(ab7`5=Sr<&~E0^ z%%)c*6U#cKTV9L&To#ZenKj4!3UyFsiHm6?)0Qc7N)hvBs#FCRbDO!%$3=$N;G!rB z;$jh@x%{h_FJEn!0wk5s=Xdg3#W)KSxqHlJx(r_I`==aI&H?mZzjxloR*)q}SNI{@ zF5d>jRC7z|_V7|0O=cwGpvdC-#3^*d%4_8r>B5Vh(Y24{nTYPrBxjN{ao^a=!|OjR zG+xL=`aGGy^@Q{=wUYF)(vwr)t~~tYk=)DwWO`D+{ z9iKk^{-3cjvYyGSUtJp7gDf%r_%pA+w}fn+t>4*t)m$t={}92#^9N20fJRc|$GzPy zc}D8+Z|&UbQNrxVd{TP&DBjp8MLxWduK#ZRwCmJ&@f-g1!p6JJ%-xAC#X7sV zRP>xJ-q4>6GS+BUWO>zO5&L+oR}d}>AIr@>whpi)E{Pk0^GG5)yVg3f#>hVd#!4T3 z9h#*}QI>`2%$gA%6j_3~)FDxlzJvzR_1IrdL}g^#9J3W-lybKGs$lx$+rIIn~||z>Bc`o!PSvR@xRaB6o=EoYQr~A=FX?Z zwSRdmgBG4@Vd8F+c#;vuq8q$hbbqld=sN?AI-}TTVcfC4iP&AfeO$DPl;=#eH>zL1 z%cG7?fx6htn;lls=oD;mZ7Mb$=(I4VsEHJummT`yR>8@oL@Qw(3V+UxqH@_5-(Xr| z)*#aoH4J%toylqOBHxuv@qwjvln2&O9Xl)tq(E7MASK>4GEseRl0K4&!Q)(JL_St| zdB+5!D&?}QHyj-8aU3+vvWV2v;lEmHM)?Oj>L%SqhG5(C&Bwcmf%#VG`Qk!P2`_g0 z<+GQiR6qt;&J6P92|bZpo6S8aCk9a&*;u(?@Xq`9B?bsf8a;c5cD_G0FZzkL$WAR|0!`INVF5sY_P^Lgl4-V zg8?vD$C`;Va$-x+%I+R(r#y6%risi_9PG}@3tf`cu!OTWQU_o+Bo@%oo;V38bJm5^ zfX%862-}pUJpss~2pH&=zjerJns6KkFoLrI)Znony(sHTtc7RS7_G7jPWavFu&;zH z*(1%*OQVNafAu6x<-p2LNb@sNX=+B`#AfcErM&#IC7!P=mUHE~_-zsOo zWLSx^vwN0bfeM8rL6mN_YjTmVKSC8N!+efBjL8LjO?2@cZ>(u*YA{9X#m>$=yn&tQJR(yfmSPngVSn&q00LT^*R|4+C?JrNlbCb>^N31dX3ufF8T`?+riPaV)p=W!eX}5D2?li zRv0RekVfTuD0YF`E|-c&MSDyh6Lu8;ZUGukH>yq-Y)63kREi;3atsRlpoeq0DxrB_ zCoE*CN*#%q4HbkGldICl9Q84>$K=}EGS+!#)^pT=oHy zn+RD6*_*3jZ^*`M_CD={54^Y!F3CRanQ03JafO?`UCs8F(#|>aJLfRdPrvi~{bpic z`L=D__z%e!1d-uKO~2nSbY$)(nlc3kzu+OsB1oBEXvZ8jX=1HtbyjgMLF^F6+WcH1 zw|Jv5{tQC+|M;c1@dGm{VlA>3%Q|EsBN)F8b@1K8<%B*+kSvqfts+PjBvc&35vU)s zXm?^D>WKdk;F=!<@3d-YWE!MYA{&PV;6NY?3EM6N2cV_2&>V@6NCLjz60$I_t0WXG4sj&=-CS+>A!ED7 zDsogz)-&d?8@dlLbI)0lU@x4C%#GDOJbX#V*;EgPZd<+`h(tSm_kI|NO#SY5WbW^g zysL~dPc_;7q&sp+!F-WhxxFa|GVb4)AxSWiV3c8EN(?z28*cG3X_8WkEZC`TrJd0e zGZt$CS*fSxT<|C_ym4fu-(0L|(4&qxvQ%STsmEAMi|_}RDFsbav~-UcpDfre9UVAN z(KL?Z6fK~I;W!pYmaeZ?*L*{*AWIiV9o429T0j=l6kL5!`IVf;dCAA|Rb=Hy3N+Ok z6IyHyvK{|iUOqkj^7XVO30YCsA+PU{k0F`C@ksS=N7~w)QQz2iExv)p&~QlCvO}-bF7+LL00ebc*I6bWS)9@zP)+iL3813zx;u>@BBn(O}?X`@yd-VeIH^A zGc;<7cwu#muzaJl%BeSC31o3wS-XRNPIPoK6XAq#6mWyeDfglup3-Jbi=;U32=F7TvlVQlvKq%qt*i;!h{2N-5< z=#D`@J2u$aI(aKH*B!nV4F>ypmGupev<(Nop6~{l*^2SO{>9rABMX$c+I|8?oj(r4BcWD9S2_8g@W$kHXoeWqqd zfi#XR07k9JDK|=hd7_&cH4a~IDA7NpcIpwbH%XrNg^UWkJX3W$X7ZU%j4A4h*!R zgTcU=kd9;~hL|UJ=I$4d4#8mWSfhIJ><>>Do+6cnmbPnBrW5Wl_Z<_iks`uf}iTf4xNA+cOCoAsDg}}h?#^ZqjRZ)=BdI(=d_F(f)yzV${ z=aIEr%TEpJZ+cF4OHt69L#+k|^o`m&Q+e$kd=0XenK#P^ezgAddb%SC*;~U-hC!px<~F>L$?DU02@6JofqSAyM7oz1SalY%IUZGQD3# zCb~z*C-2=2crP_Ted*27vC+Vowt>}u`P6c!-5Y&@kY&SdhaQ<@!b&hqWPdn~Y-&{& z%k0#jEZ1MqoV9{PEK6NESFZQAt^@`ms`%Kk{T^y1{lc_*Q>8H<7h!qDg*i@*wsEgb zgJJ5uSWdpnU)@JJi){I^3pfCY^a}&6m7a<+DuFD(>h%@1hX4@LFQ5jpD+h8K_gWKU z;>ZH3x~04gt+f`}=WpMIKgyAy4niHva1fHv zb^zs2DEa3&oenwaILM+V8|xIDkt_cWSyl%b7CG>{A-b9b2T(i!l}tprTJ(b1PBI-B zq&EG+P`ZQ?D5X`jQZW$Weu>1Nlz!Q1%<1O~REdfKjujOFUOq9J!YCZ0FuuTm0mLDO zvL`@M7`DD*fG+-0IpzW@MJ_&HBJLByK7=u%TGW5>+R1= zZ|9TJFLb45SoDe=Tl_DvooK*hz>=MkOtsb{Htw+JGAC#`xrR~5rSD>-IQ>GJ#+#j2 zL;+c}Vdu#MiMlMm0VqsxP!MEBtdN@w*M|%Uby;Bxv)g%Ocld!Y3C3eC^Mm)+WqCJ> zj>W}m>arbwUV6K<G%S->+057^{50acOz{R|BnVlBJ9%+)GO_>(?bSc>8Vj`Q z_I5)GpiSFHVA;yGmQ9jo#cDH)LOx2xI)^Of)-8Dsg?Zh=iVcTkt#eZ-g@1YFhAp}b zSOsG9G&WhjOOumE_Wwg?-+6Y5{LsTxs{g`+T$_3xOhs<$NiKi8ZBMN$$EC5RnN8cr zXzP(hVS&xo6?WOwNM<+sc@Q79Z7dz{aV0)E`_D)Bi;iduXN;FU)mA4?0kS!2mrVVK zBW{BYW;eZK*+f&;$86?UfY$V3GZW^lPt9Vn9BF>$L{O7rNUK)X*20Iw;j5SluGpmbP??G|hrBHY)0_RBN@3Ku5Ae1X@8zj=XrR1zF)-um)|%W`2bWL z{kKPk*s-d{|F@k1ewo$mcUA@C6#!QmTKaP+%0VKd=XCiyn4{Mea2IUMY$O zwDC&vQgHlmSiD^G4?Ow7WtmT|A0qoNs;HYw=6__(2g#m~2YrX^#HR@T-Nc9U{1XaO zpJ+Vx8IZ-0l7`rqokHPA1wiBEgc7Sk5bP-YUk4NLWgc1dFM}X<`+PKEtY{b^y!q5g z5MUZ~Yxw%s$THC)UZ!+%oTlf*f=eCEkG3GJc-s;sRtXxx8ysCDPqpv2{2B0!qAlV& zdTy}g{Bf}W6+n@6i%Lwl*5ja^5D5sJ?ma=z?Nc-*qJTgURo!WVm=h7-4+3hwSfZ*> z{v5gGJ+eS={tIStkY%I-UVsOJRZbv?vV^Rl$*7zHe zCgX2HQ{KQ6H{SjyhjZBWEcvtZd7oChPPD4%RNGI#G0pMKux9pjD(>xdH<-PS(9hJu zmXE1R&T7{27O@H)II-YvQMHNp>47@-!sGyz65SeXtZnPW@lo6GBGG`-z`QbK5obb) z?d@U5keZ2_l+KyUD={8f^m~=NK~<-!?->)SmgmHynrVM0A*{Y$r<1Gd9UFmM1A7OE)uTDB?3oUvDR=t*BeGsVT5;Dl>lA1Cupl45u;*p z+ZG|N^L3jS1*2m5iHf>T5!h-FpJ$atkUQg(=Jv13Kl{Za8;wO1g#;H%N2A;z#v@CF zB=$g;ULo>RL{hdP)f=uET}?^7G@emd7!s}B)w2Y})$cg2Hmuxb|R@VPCFFy2qsw%e|tX zU0V&9nV?;5mFVwU8InxuwUX7DNHRl`>@V^AOVlst`*%D|uw`QLRFxxGV9pr|D5!zuKTUYqSeH4R@qFS{^X8x+J6_wQj7Yz9o#Tx>WfEAkFBHjAShyn!0T2rpu9iCP;HRj*BIdJs{T6SelD*v1lw`h!tYs zce)Td$0Lhgni|qisbxc#$6xz3*NnE}s@1?0EnUp?hSC0v=rDCPpq~NRucU@=EAJV_ zwI3s!22uY9h;$;#6`GJkWV^J}T8W{<=5PS`itc!KSL8}iKXmQpZ~9FKnX)ahPc?3K z&B1QR{pwk@-S@Q3X*DD*Deq zmQ5+vHdIaVVQJ;niQf$u?3)3GRMRAnmx#g#mtF#~fo4XZm5?QquHf*d zZ=1C}H3*02^(%%ene6!5MRrs307mJruHdR>MPVoK#*6TSOz)6IoOw9Qfzae`p#Vg3 z92{UT3cLrxp9b*qPOHK%X`JRrfCL za%Ozu9ELxXeDfi)SSq)daPH-Dfmkk=pD3-eWRiJ2td4{<`tSPMhgQFkxx@;WEz;dKqA1{6r%eSnpklkiJcS%*@F-?|bIy z^t{h`&NNxy!v&1my6&oQe@C||P@$V69p5I+E~9nEI2$*)yh&?k$xByPqo0mEBMCA- z<#0a3FH@`fDQ~26nv90HXI*#id81ad=5FA+)qSJUT@jv3+C8;af3r0J3i#}yu*ntg zte?1Su)Xa3>ig$ahK_EVJwVBEQ6`>sm0jZnAM{-ixO7}dgEv0O^I%UpE$|vy2C|k~ z(=XP?R(*YI-L+1Q&eN>(utZglT34*CceGvT>oqz(Ogum)LlwHfh}oSCMfGkHt+CHV z+-O*TC{aa{M6@Sq50Dq`arTfmq3vbx?ok8X>LQ7--ZjxX8aDr_%`+JfRNOVyYCC7c z9){{Jukkxf?vX~fv2EfXp7#1-t-swBpE4AiUGa_{b?t=zl-Xpv>;Jy>ajX6$C8vo^ zV_8IqB~$zopXMi%v6LXBxOC+PWdG*3xuW`jacZKu$EclsX>dhsz<2wC4W6aCDq~}S zvrf}ek7YqN8jJg4{9-D0wu?_MzZ2F{9H=%e#JE(-$Af;9ewqMe-*(yz6*^DxkCC~a zV(kp)He9cF^p5JrTw%{7iW1ar3=EKJ3jph-tIngFA$tI081@naj0I0j8p{$LGhQ;{ z_Y(9-bqPU(2SE`iNKa^r0J)JGWJ?4Al7EoSz~MhQD5e%DMUY}X0L8N~%><=k5rU>b z#zjCuEzCaTpctr$2KCag5}GCnxCa&huOw&<_LHH5YcOd zd2cM6`5=?hFG>xD7=O{TBsqGuQmmPPDG(*rOlhjWBSD%3yeFUp7)D1>&<7O62X=y> zD45QRp#xW?AjPByDuThUF6ag*7KX8a@)X>JO-Q{UZjhsD&P~E(1{49c5wI7c)qqU1~O0) z>l0WOkuj_=o)9vf#b#a*snfu~;EotREjlc@i*GngB+vRKfNg zrrPAg2P9VuBil-HsZjh=`u{_cYh#Hwf?-JRP7^u3k5{bv?7ZyJss}br)l=K)O^`j3 zllse=a?Ua1tWxa)sh-uyruO8DZFt$Oc6jqtLJqfgB@$HikwCMK;~B^v-iDVgQI*%xdI$?@BYk6Sr<;)@O? zm)(#g6y@85)P$COa!5Byr73PjUY38SDat>)#sE8d6gmFa+M8LBmFH*_>{3>I!IppE zs!ba9ioIaUkB0Q=CnwVjdE-m<$DfJ+Z^HI#WYcZr@}1dN?Y5NQQ@qPlRadaRs+Y}% ztTMkP61U|__&-^@XU8%cyq%nCKo5Mv&*QBE5cUOR(G&S8(*E3Vth{tLt*k_v#ooau z#gI}VSLSVs0uc5IWQX!u_KSVfaBFxs6^VH1F$uCed37R}s%cxQr>p>k{Q}vX5Y1Ml z!A}6Pw^7J!Qt%bcNDCpWP7(Nq8j+wjB!`))1gVT%NtwoaYVTeM896W+aLOvN*gh`OayKG7| zMKram&`K3f)8*|%(sr)yHIy7GZ!*KR)%Y1Xs-l{4P2G6uYaFU(OWPZWA|~M~s|FWo zQF7Q+J4K9?4Bu>^scL|CN?k&ie@7EaHAyF~;^Jbo&5+%2uSHZD!p9Lrrf6xAAd3=S zpUX?ZX(VWr9KO>u3BEN|G)YWM!W-VA1<39%$TB>KMbqbs6tZX(D$rQp(xf7lW^NSS z7)qokQ#>O<77dzv*Fz0Kb0G5czS1^q*H8aGy}s}*Ief7sZ1%^0b#YH;NizAb@i)D% zU(I_*Y!n%^u6W;0uY0W48Sio1A^Ik>`tw}e0xDU-pZAwuWLICX`^*jr*vS{oGCd3aUS`H?oQ6>r8T|I~`otKG_MbQcO>R{m3wNUXDhptUG<@M=) zx$-PoGnWkhe6CgPyiYfuI13jk6(GBRAd68@;Rc@RuC=x07}TUKFRd-66Yuy%us^o; zj?F+;6%YKT>vT@cd!6I220gF(LT~2oT0)x0zzehCk#V5c8=TT;w2$ru$L6973Co?@ z(O_S|(H~kj*xMGG+nw`mgY%Aoz;9z6`U~?lx_M*2=Z$6B(LWt*?dYp~3yT08t zcG2|Ukwt%>xcXBsUbxEcC&3HzZkb2dhSnzoi#wTM{PV3u$ z*Ub9ftkk{K|1|KTyZ!ork3I}dhqP4EF$nw z+PSp2d^Qa=z2N&4gL2pmWGUZ^y6|*ZJ2t3wg^N46g&)GM;E1*Hq2Ai@!09nM@8fi( zvH;mV3t1T=+sYUmIA?c1@0|AOVS$MCv9@z+!aN04UYo|UGGELm@JoD(KjPyA!N+p~&!q(6 z9410mMMdhHD>Y{vbt5N7eyA=#PbOchZ=EmdIFzVzH5pD-GiYYvR{^qn5we&}ufDW( zV8Ad48;!Fpf-wlxxFMnz4Q#QZDviCAVGKa-L;*y?#eAX)4{FmST_HhAT03z8A4-eD zJ$ML5Me!kA5lG;2P)t-4LMfwIo|J@oqDE2LptdpwlTd>vBw1D_Lx6v5MkN z!m0|q?B0Vc_D!31&}Oph)X0wF!`zCr;a!Go$;YP>hE6CI4f$JP_G|A!_CWuG%H!DO z|FC!dF->H70H62p#iX5ShR6`sM6_HgZRA&>qX-D4)LP^hQStSp*rHnAe?GK9S@Ph$<;9lJi~lbtFy! z=4vcdIc%i{PG@!Ac+q#vc4BrWf4Xm=8o@0fA4O|mL` zw#Dk?PE1&ehO>31QqJnJ^wQCeY{GQ+^vJZ?V=1?Fd29zc)-(Kv`_p~e(L3X#x2F1P z$_zE$j=1_dCi{xGt5aOX$mJ7LmIIa%SDVLjkr0sGP-F`hkWHIqt?f_7hH_^Owi&11 z?cjz?<*aq8Cx7~yhqX@XM+RBjhHc02m}@&{*}{p_UzORIjU!fbF^Slgqvj_F&oCR5*2 zwr$WhX38}6d%7K#9#TMdV@FxG-P7}Qrf{GydHQw&b+`Xf|5V#K(~y<5pP1;s%(@*j zX)n08yPW;}V~4%Z(PFH#^fw;Sw>$sd@y$d_UDs2a)x#chU$z<=_BX0*R~_8rs7E$= zs<&f+vrhIpEL|_m$NHUJw%g+tcAx%>Dv7AKWr2c0v`i|(2qmpnA{<9DoUD?P z3ZPcdQYnLbJE>7u;)DVbDUfuI(3LQ*k}m!&Aq}y6F@vnSpujsXC27F_M2V9NVDsJq z;P?LUD-j8)i~#-xiU?r5PrKz@CF&zL`9 zH{TEcdQ5&tM}NwP_~N;X;p^eA%VSXRufJUY8LcHCwY6FTq668FfUMq(_Yy)zc7@+W zEB80Y3bLAlf4_TMGU^&t4ZB89H$^kDn^VGs?63G%f~Me?cW<*?9TjEv4wbP`KvqC@ zQy|+q+Aypet{7_(kQI>K9LS=~iqkS}#rfT&fNZ#s_2-n^$cbP~A)7jEAFdg44XXrO zHeAR88B{3A6imiI0onDjWsPO7j%>k}U9(ri52OZYG;`yrA6bxUO3mJ%+fb*g+f4|_ zt{vI*(Su_R!z!16>{^K(@APMF8V8`$pW%6Kl%+DWRC`T_%lEdFBE&>mV$3i5%exDT zGP5xw+iXs!SBWehYyr6>*a;*-%9vn6$4^fz#;;fIFw2OC1OQjrJNW;R{g{GYLHM^NH!G~(yP7u zB?2Pl` z#7nTiICMs`v<&4etro13IPgsPM=QlnjDdRTO{Mi3W!H!-v+l^|gdf=_&%gWX`FGzw z{Ca?%OC>k&F#rhCfCj;{=mq(n3A}p2DutqY>>(R?{T3su5s(cTS%mIa0TC2kLJJG$ zLh{QB$gUT%hyWyD_}>I10fGEmc?-yXTr~dmEgGc31Y|b?S(tnNOoNtAFS~LuGyQxm zC6&t&E(bA>EVCT4@ruwTUO+a|qim{nPyLOAx-8?NQhintY^-lQbb7x8tMxkFlenV>h|5wLR8T;>Yz2Hq8o~oQW;cgkbrDBkp&QH z0u2L_B1rEjLeV5iPy`|%^d6!0v1OZYzWU)OKeC(5FgW?2|9a);X6k8K=Kk!K6UNW@ zOG4kZTr4On(i*QOI2-GATJtqRKsJ2HB1(~3NyCWxWiPyW`Qtxc6>LIek6!=TEBB64 zgI9G~re0^;7d*1t9b6Z<%l2F`{#kXdOLfC|ixiLz8?uNvR?q2mG9t3be*gB(rbHGs z^ZDCPV$`7f;`jl&)<2vgy1JyK^PEU~j!MsHZOuA#2MWlB4OuF;Oq43qR`@5o5khvy zOYO6nSIwKw$@N}f#_eQC0(yS>MR*M-5>lksfFfv;@>^O0vf)4$i0k8FE?ZJ`g^VDw zTOALde3g|Kg~+OtmP3{i;q#(N!bKwf;TQgy-}fG$+y5RHF#@vTK^CTRyK~J3xj{=s z6xq&v=jTy~?6%imCM|=kXa$KK@DGqhG)a3O1Q3v2hf%gzFJ%q;()J&SD6*b`Lg(2i zME2-kum5g2!$7kj-s409(KPbCLW#hOC8QVj!Qd;rM;6wo4wvsMJt74HvTL_x$?{Xx z^0dQdDTpYtozaQx$uF-TU4Hs?ffw(@5mHp~VY>Q^pHZSxjLu|9y)S8sA_;PFBo~5> z=Azouhg$ZMp*Vt3PAJ}DaR?4+6qCq8MVZdfU~VNN=`o-@y9=F1sAxp?r{$3?`0?eN zH!oimz}>keMYX#_hKJ3dnDIc;=yE_Ak?K3vicF*|!LgMbx^PEoBPtX0I|Ep=d{md-+$0Jo8 z@3-FtNyt`oIl3l0jm|D}y7y4R-Dae7nUwDKlbWR2fp%F4Oh-HvQB=DudDHOvT)hmD z4Md*0RID~LvcZ4Wxu;}ZhG+jiQl$3 zUt@*Qj=xk z73Vk>Wk7b_UI2C@2q1zGsYLUESe_7JFj1%wLy9ZmM?e-MA;B~O5@QHR!u%p@|7P(Y ztB-@S)MXEO{4)bR))xWBTsQ2S&H*Yd%w0I`xIo0ot4lqXtUbTQ1evXptGfusP#EK-By3D@v8W#40JkIyPSL4D{Gd*qHi+Xo z9Jh$WxI4CFUwvZduhcq;)fpd&Cy!@XP35>6*I80$4uqc`9EgT#?kzm+g+z|P#SeJ7 zh!AKZ@86(?W<_gO6Vd)j<=I;O(8mk}7$gu-ko^%7X&hZgZPpW>u1RZWM^_bWsXyA? zGHnEDA3zH(@l~WXQiB`cO!wD|!PO^0{!&9rOjK{WI@=(QZ~u|5$kTWr8uoL0l2_yF zm4Yt*<-fdq_W^O=**!KqHskx1XqWxnmD_LM`|Q$;@7mmS*M~pvQi2Z)j4Wu9Pz(;y zLc0Q}A%Kd|h+PTz(=p8PJG{B-i0lbuRkBN@2}G9T2clubX)~-|N0ywc@=czeYH;iq z?6R(VS3a*awT^fk*M_SGhlj>*6AH40M3$j8s+hcDF<9#&o9M&2uE*uaVu0MScY}|f z$np3U=At~Z0FXd$zicwHuf`aNiUENt=T6UlalO|uld#M7T{?N~ipYiqWCNUPbBC?qqeTD;UfM*6 zw@|Q5yj`D{q!_@yHgvgB-?;s(1SOK|C9+t}h|SkCa-1kGvKM5EY$zNSkxlkIq=&_* z00Vn_o2XuK9@=Mwh`-hKqP+~S)-oy8VwkSs`pzDk?O1x#a;eg>c;4#ET6Q9`M8lcZ z=2qu)QIUNhdt@~kkhL#8uAyWt#Z?dmckk6$6|{_MtqHjUsxVH=UzCYf7~k1ZsUPQ?0u0>4Q*{Dma}s zL~U=9T58e`L{=q=t9{O)p{gLnu+~6y=ipr4*h|aU}o(rhC5%iYzT_WW&(| z9L;9ZWfSR&X}NU2Waao(3L)OC zPOiwhLeUV9>|stPjqHZOU~lgm1O({KL{d1yNC*?R{C~)h5H%^TgdK(K{+LVF$R6P4 zC6Rrl^v0ozEtK9+kSz+b{Fe+up^1u+FM{%mtTBek71_AB%ZBFnNP53CIg3)@sQ4k? zej}!D1E>ZV%}k93FeHgFOn&OivW+~lOC^Zx7DYj}xFOyRqv|v5l~vBxPG9p_D|y-D zG4!8AIcFtoMu_%q%)W+f?pX=a$mX7vkQ1_j<;Og-@x!6GHtCUmR8q*g_LL-~b+Z5f zAOJ~3K~y8Mpu(oE^O3bRqb;7(dIy+r_Bh7I)FAcS=vS-LKbJy?clzLlB@vlpcK6bX zogdTCQmb*vM@op)YUvlkH+!KCu`euy0--qnH<0v5VRh*s>kUV><)|3oK4VMQiI$#P zhhwnE3hJsnwL>Fjkmg{a(|j(1*CnkBD=sF|BxKjEm|H+rY|gsn)ma9&^FkY9KP4a= z<@JMoX{-IEf-Gx!8V;1D#lQ?s7#h_k|G_p`O#`!zYBaQgT))GcrYmaiu84{Ol3ThW z3i8PETEXG)Z(OCT7`WnG#9Nlg0)#;T7yvQE0I{|N#G3MQ{LP0sGYBW5Ut zZ1w`(xlPinO;V680J2|}8M0XO>0wh()pR|Ut{iAAx$Ls)tym0rWL?QmQjq<@yX-%M z=z<)Pwe!e&x%ogeoKIvCHFm|;+^O@88lXhsc}vQOMW@a_DZTK#g`yIZ-U>xR1Bs!X zS$JMSHW^t?8_GAbDnRz@j<=!SdWp8x`492-hN7WB@+1Y>AFLS26j>x9oA+H-Jb#fG zf>L2c1#l(s$SS&QxghI)w!pC*?TzOf*`j*C6lBXsF?fmW+LyI7k>%nFvU%TSSA#0m z$}ZU<%L>S?A{ezhRB{;;QX9LLAE53#TX1?00RUD34kH7;|jJhe}wN<NB0bo)y~V!wbfG|QfD+GjWIv%J?xKpWVI`A6c5=P zvl3X#slV?CWY;_KmRDAS!P(JX=N~jpw6>3!9B_Peywzuuh;GHVj%-js))g}r4_TL6 z>tb_2mgbSAIE{9mb1BG{TvmeLchuU`Xl?JR9U8CdL_J3u9Ic~`>1~54N}Vn`T4Hd+ zU*ld~b*p#9ZdIt&v~kO;vkgkT&_H;3d6~kquCAEpk)7xIem2h?N_rvTk$wePxg&eh z-%{V!X4QL+o2~uuXn&2#)2+!hT`|ip`zWZg$AsEI#p#t0TsXDxY{`{%^_~Q>`|Whn zE-RVSAYWuzPFI%5st|wjA|OUGLVGI=VG#0H`AG^7uC8t%TNK5B{OY?u|20wCEeo>W z@`}Ooa~EF9iUG^(HszSESV$vR(UB$YJ^1L3>CNoA?EC`1%Pv2EY)|g*QEJH2|7Y*) zV%o^A06h0b_XT@KgQ!=j)kd`wV=N2*;c=il#5OTDIR3%rH(rba4nJX%KpJo*HlU(a zcT}y)L)YESLsKoPL?60Ro{Z#XA1pv1N*al%mEu%UrD*fAZ++-nA3Eb-jGf@b6R*6? z5%SORIcKh!Z?5mXK0a4OR&ZpKHANOdcQWz!De^>b`7V#_%l*SW772+gqGV7;Q8=mz zvKZSYBKuyDeOxji{>J9n=Er5^|ZJH|~G>T&?e#714TkvX1HtQ`h z;v%W|ND@NAjb(>oY`j6j8Hp+0LL&Qlt&o)~Q>b!e@9@YTC%!Hvi{FgHahvd0%W%j zd1UuEWQ$54Rz3Z~2#7(5mrK&9#_ zokNxz9>|4Uci0t`)N6myhXzy%b#;vYXR0^xkw)9l82KR>~pUCL&wc$jZKWMi1lIm>hC?%emt>coBrb4PAC>VQl9y4Q_m`3L z<-AK8^z?ah;Ox&+!JnQKa*0FsaDD4XvP=61PmhLd3i;)AgRJWGBCA<*!?Eo49_XHR zED(-pz}fhnV%glhQ!3?jN0g8E2Bh?p{nuHZ;C+I!OoLL1hZ28k{bcXM?d?N$>(QgX zZf}(so~;*T6=`JC)k98QFKb?C?RBqt1AXo_XzuiwM;C*D-v4&GYWeRDkE{TJDwOZ` z9uiC6RB*g+BY%4_{jWz49(`S6{6dQA2-(e3i7W<;ihfKDRd!O1fqtp9L170N{?+ZO z<<;{Y8@81E^X(eRtAFndFiVfWuE6rqXRmJm<>|Rg4_+OA>He8ILAEK*Z>qC#Dv`zb z(ghH4w-8~76TrEkGXur)@ZmtIl+zWI^dyHYolMZLv3%I@)*;{6Ante#40lt3H~A?& zu#!wBAYIZRRNcU!W`*a|$ZnicWXlO!nJJd**Jn!QCuNnn2UrH#Bz9DJV#t{am8IS* zlh=7<6DucsPd-bmOePb_$-R=0)lI)3f!C<;d=}YsDFzW)ex#I`6hN*Zj20wNg@OpC zvg0MLKbcI@^wLh5HdlizKIn6d^(kvncwQclU5IUL#@C}w@}D&!5n0Se7Yp~fTPo#r zN0lfasI2$9gnXknYGbg+Ieq;8(|gyYZ;7Bg*~yCt-y%kds&7z z+h8xS>}H&e$K$(kF~wVuZ@-Qnuv{gA@2`q?+3mK4_$ooBHNUp-JW)yE`4Ybpqc%;t z@I28H+Z@_pM`**#?z%9v6eQI(M`BuD;elPwetMTfFw zokV0sWZ!!F1xOr$V}<7t1e6SNcZG`d00NRkKn5wiV8p%LMd9Fx5u#xBHfxlwYR^4~ z$BZDAP7o216_FKUT^3<3jiFO1V|0pK_Y3U0mXRr{rBm73lf@CaB=%KI*4oqWAu^{7 zDAj1yD6{LJQ=KtF%pj(=Ks2T1OtX=U&A_QVr(VS@Y9JyjA}a{8Ks9SoyB(_G@ouYD zmRVTnTqH&&T%)%<)`%*7f{hfiOw~3W9UG@ZyN4 z+ikUukQt+Pw`;t^rFHAABC;Z~LVf#n(_taK%;4;zb2YQ z%cswd9>~isQ%9%a?)+7sfyx*K&8?i#TG+9$3`Jx`WQ9Q%5tyWlXpkXbgxOgW>PH%= zV3bsg2Ih@0BO_I4pc3S!R*krkQ6UKxlx9yNY6L`(QK^lL5q9O?9d49L8jWO7g+ydU zWCco#L21E`ZbHGQB5==99#7tg-=?BS)dkW&Ge2eoAS{XmP%e^)tZ;-dZM8wR;*d-& zk*%4C$QG_`?z+afOB)sz9J^3D{Uek2RWJk)r0q+ToG(DdA^Ek)qajhcKPl>Zvi%C5 zY*rK!OJ|!E@12uXb`^(2m#yO0KJZbndvG(GvW4TMPuof6_w)S}2Ifgq_=(G$dR2~sQb=KwTf zMx9C)Y{r!N6N6Acqf-OWY**-{M?zA@1`<<}!Tx-sL4%Y#FsTXR&Bj7V4b%iF*bfL} z;n+w+*)=5hQczdGhatG(ASp4DMxqe1A4rW9Wh9s$RiQx|T+u=zHIM{hHOL}=C>jmP z5k`IX2}NO4m(9F(g=8oU$QkbL+*coyIL<)8a72>~eBk|y>(+QF7NZdolm54smPbn%k_Ha18sjWX{kq z8`O?pbIvaN^!cj*?~61~c!S*wjt*-s=@g-kcF#ng(b~~D>&%6Ox;bO`#GtTemVBDqO7H^-jsxq8dl8uoVI$fsPv(iNJxsYH1YcZ<~3w@)J0yaT!&s>Kwpu20)2Xn0^o$VuY zmtAI$IWS7)yh*RxH{J?IJy)l+|7Y)Pe%nT_0M41sAgDM)P-s;I0oH+7P$Z$%#|Wk^ zrO+ypWy=yILmzb|FQUMf3jqneh_Uj<1!AMHouWXRq}Tx4#7=h;WDo73Xpo+A&0#Mt za_isGzoEkq%aYqoyNS^#>b!$259i~N{Au{+&G1dXN}N(wu4*8q$5Q=$%SiM}|*6wx1r6fm--RE5oyy-XWq&$fd8&!>nkuxUyAQ?Gpm$V<3g+XEu|~t^VdNfpA)j388RmAu$u^_uQ-2Y($UT z+enw zG`!XS4Z+Y-tKF{0YU`OAI7T*}X@|9I@+|xK#gpFe=9an-)?R$N-kMuZr_=G9yq7&i zWPv7V0;OGDn;RG=Xeb0jvM4PUj_pZN!5Y%mvc98Pafd`**F{=b#$B~&`Bl(-GehU2ZEh(D zIF->BN0!T*3HQtX50S;3w^NV?L$)yridg<}NA|)+AD2@Q!M@BRp~?9(V~*_E3ui8H zd22cAJh9G>f+}amy_aPdn#)+p*|A4<;`|S}jFFrldt}cDBIcIp_y}%{y)Ap@oR#C_ z{qYfmfBt`f?3muX92iFbwIfAHtFqK|J819?>Yh&}K%#r2Dj+SBY_|@&h-CI|7_g=32@x`^3N4Fo|{priyrNd3u%N}HQ zA8=$jvLiP9vM>Ze;ZiX0LGa?5PeK4&&_{I=Tzk0x{DwNFb^%BBeP$MUt^w!$TI41` zdxdT`N&xp5aAe;j8GK`$Bg>KH!ja|3a^c9n4q0?*;9Vu;#WWd+mvJPHX*eiCKQHnQ z0bwZ1ae2Ybo*PPZ1Ew5h?N>5TAM)x4L&;9`6;Gjlg99}IH}sX;ns*dcOL+7Y0TIn^3SYac`xBu_am1kbFSG5CK0h42XReyOGdv z^CCe+2Jj(K(xGblv0(sJ#40dsj#NRkEOs1;CaV%-!`>NB0xD+dz^}8Sq68Y2DX`ia z!0)pxd=46H@b4Ee5}^f=D5$ChfP|z5lGw0clBKLFBHR!p$pi*#MgkUA8xn39@F#`| zbyc>YB@tv{gRo)-wA3I35CcP~+vnt1wzldP5dA8GjB<%Up9t)8Ms{k0T_ZpMBoEIJJB!|}FpJB9on9gTc_bBfkR^GAW>VdDvK61Wr&$Y$?rJkx zn@U_=-l%B#z+7*6rc+*+4CN;l7PedMk7tYFc)priZ%$Whld!1^lTxg$&7!%5M4aU< zDfxRSXOW&$&%WU{0r(ygg#u^o@i&c`OTm1=e&lrL$?e0jB! zZcop|=v1ND$p^1lNFYTD1Q_W>!Ykc$F`OzxggcW)ByCnP5@3?0CKZ6_ayC>@YBSTR z%wjK|z8;#72A4I&k$vaL2JRim{rBmSh(R{0(vXnV-Cj0oha^CXg)xW zwtuFGvQ+V$03!LqbTSwWLaM5E-g{dCtk1(C-zIP(6CU|7-2jBPrD#Vq zwPTz*VNC)13V@pzI7#dgZa~4cDZLLb(jRa`4%!)QpQ-Kihsdkc4v;8G(D~2xrn<3K zffI1ij(iA^ZIeQ;griwMY@nUwG|B7$=%qSG?VyXEhxWK33qZpOLFgC;uIjFy>?%0L ziFd%UWl4aTvJ;KLQ5`~Ah5^D2gLjWCiSK>KefWOQ!0$6cK_{mG-d+K8x*Bqi!3ZNe zv3C$Sxo4FVrUhC8Fxd#izSHK|&K+@N-$!I`eZ1b3hlthYjBTnfNx>i&@zx~(qiYjxDsh~ zk}C-c0#`cgCTD{qJ3_XsMTYW&sD*bTG#R1OQfQ71yM1NK4Fits$aoC4i{#`$8X_c{ zFknI%IB_+B3^^Jc*^%+f(&c~Q@xO69q1R@7Qvt89Fn0h0pra+y2p%skfyiK=IcH&tU^ue?*!8swu75uVOz2KKu*^>XOk`l1 zZX9LDZ|}OxoR_t|X$SAGJBvW5=u9Zso4I>Uy1|v9JAna2;l#Qb0Nk%RZ~2P*4SF@0 z(47Mj+*ZA{Jr0GE4*<|#T*H(!Jx`vSI!G&!B-pSJM|Olf2Jv~%At1L>UA0WE%>I+T zt9xx+$KsYiFE>$8p@v}}0x>vF@Hmc5ln*YR(E1~BNJ2gcaiGa0DFj?MG%yYG(mc4_ z$5PnGg?-xnM^|z?Go8!K-PxIiI~M|daE@e2EGa6-u>i|Doz7CoqQ*tBi-jT# z3`lXrDrCeIv|0$sM38BONbwcXO5T2y7HAPD8<5V>q5!%%qP3$GF6TwV&6ZWW!biq{ zC{vUJkwA?uW{*V<%Ob!z3>Xqo)GaDqHuGRj!ZO5i>Jdc@eC!1bg37C8af!E3@n|Y7 z@<{?M2Upw#l|ZR7>BmG6nG9>cyoF9rJVN&8f$VAN{y7;;cGi*N?RCj5tS^~fJ8L0% zaoE*sdvCWYjm=YaQ@)+n8K8oWqSN`kL8S9kuq*b=S_c5p+5&HKYLM z!%=7DmFn7ESlx=lxpeQ3<3jIlGuSR01A9A(*BiGV+%>fQ?piP0_KQ9C*sZ^lov<7B zq?MGNY5S}AHa^^i>fSw@oosKX^NV0zb~}@>Pt|I5UtKi1e!X({s5Ufne=rxuYE*wO zo0ro2sdQ&sjm>(N23d7$+i|_nep*5BC3H+Ho?*tKp(Pe5Y$2cT{<;)(V}QYR4|=czEMFbGg})Y-PI*EX(nSP2Vn=<%gO5 z_r9~@JZdY;J$rl(~+Ukg_>JW)D?d87oNA0hiQ z1i7+O>X~MMJtNy)F9#jvv6k4ka3E)FyR`6@CBh!(v%G%LbfvDBnv1q|9j`U#Rx@{b z>&<#tnA&dxznqWDdfS^+D*;gpXa`kW9#E<0d+TMe$ZQ-aUT=r_sh+p3L%G(~%l5rj z?;4AZr)8I|vFqk%H9>DrE9#YH-1(b{tXi6>E2h63#(rNKbn{l#Gpk-sc&sVvbxrrW zi}6sKl;VRf1p_4rO8t7Imjb=$4O%n5rBA$_Q7$>_vJw(z=9m&-Cd<04_RCte?P^#+ zxf*bsOf$`<>6A(10sz&@!LkjvfRaaEDq=#onQbGg-V+hq0pcpg1i%`tkpxkM~fD-0xK|&H{ z6fDQv6Je$|2V^B=5;g{h_^k0u!f4N=O3OrtM zdxY$dxL;W#13y<;Pm3uqr^wb!V$7R6fxNB(=YWQP~EZ>Xr|Wfi2K_J!BTUn^(0LP z_Z5h+XO?ugHJ_%N#Oo=3#h6)fqrB{KJ5hkfWmoGd>f+3<=l8iFU*72nbCGU(hE-{q z1EsO1^Iy;rYe&fbKjN}aN;cAOD)*~h*nzii-Hj+vZ>wHi?WD@D4TVN@a5Y=lrZd;f z*V8+-sW?-8ZVj)l{4B#E)jH&6RT(Pk{Z?oer(ySDH4#|>sow<;+Y8s6Tl2R~=}5CV zLiR^?zp4}R(k{p}wzXa*2p84&ZLcv#c-bG+o05Aoc1E-=SGI$dsb0-;O|cdRv&wRA zFV?C*l)4@kVB3G=H1o4X>(*D^sYW06Drv_60uHua{lcsHyG~OarjADVfWKHeeoL}vMTw)~Scxo`!dk5YlR8-b&eJwo=oAxk&27hzI<9||WxeTa;r&-jMm zQv?<94FZ5E=L3S5RKAaBgwLEU9$rPlIrzfyR`3T$ZRzKlAAR-5iwG{H=P$}oLNNOf ziTeV3kYYr@5wd^DQwgs=fELJ0iJ1VKHs*){XOjrX0y7c_gdzk*c~`*fkpN0Ey8;x( zEf0f?Twv~%OjrS+P&6XSsuKcAGDB%}Mhg-KIY<>+MEqqQ^QR)G`cY9TCMPkE&S?IL zl?8+hCiv5YV_$*22?z{$gzR7P>{sP^^GLLBmzpm6=C#*?oP1cst|H|NJIPQLS{V;( zz#189@!-LEaH?6#8rOqz*v@L!#4_@Q*1CM58-|tgHtpdMGoNb}^}>A0ji&vjPL^GH z9bil}^<)|UhrO$5QCwNWse){-sgQvY?n*GAiR9#y)EF7klFpZgC~6a0K^;Fpux)9< zZrrqs%8x?)44ER`i-?sv;sk{IHAOJ~3K~$dSDHtcc zA+HZNp>v!C#|@)-CgYlj>Oyj&srkP75lMJO<~2{?Bs!6uJZhLDE7;tU1`}O zfPy5nSY>FQ7TMMVl&-0v24exzs4-^N_rpd9=UtGI?AR|RBY?4$!H%gfZ&j;!hU`b| z7`!TMOXgP4yBC5tdzv$Zq=ywxvrK8e>NVJ^|8=NYUb8PH{z0s-zL`R)SsHq?Nn_*d z4bwN8$$}rN&5ox={n=_wsN)m;VG*Qh+iVQFrRZKjdN9S zSo#dvk0qpeC6$0d0V;KUwqg|_pMushAhcMyQmH|WVp44*vjvS3l1iZQEv{PUXQ?CW z1EsWFK~pOY67kBZ$s4Cqkq1a&X&&(*B_m4ag}Tls&^cyBTq6d;j0%Mc%ZtaFwdVz4 zBbQqPEF@5z6=r;;0%xmFRRvcSbSygwDfuQ5e4fi@`U^d;3c!?CrIo$}EMV@K1HX`HGvc!uo9 zfh=m((JwW8PTPBRQYHLB1Tf(=WiPX$2?2m;m`kO=DK;yZvX4bf3xLi(rU8i|4-`>u zUBWD!qH$isp)4T#ABYHKGoaka4I`b;N-P=^les(q5T+whjX`(`vz!#1X2>9k=~Dog zeG3{05WotBc~W1UZ0{f`%E-m!BKIs7k>UA$pIkb~xrLY%d)XCY5}VT@;fMBb`Wr%4 zS-s!R?`c9Ts-yDyeTP^-iKUF^LBt|^ zhN;cB{Db4#Qgo+qgw)kr$lZB+-M6Fl#Y8+WvuDV@yUA5w|FSPzi0<5{d3)Ps9?-Rf zDXjSyOUW>rp~0(j3$%2#?M64#pxO64t(hKt#rS4#dy`hrGn&`Uhi<@TK*aLs>rEt0 zLg_FCQJC(PL!T%8Bs^Sud&822R_x?7`x zJ}VyGO8lcDOXXgR85U%v)s|-bF*Fm&7)Svy*wEEet76G!BvgBac8V#WA4{QCSy`+U z=zW_RmQo2Ar2Q1Sd|(U8j2{nc#+f2>hU`0OG4Y@1cX|p}B>VeC4woP#cX9wPQ4!)I zKt#$$*{u&DqK=c{1AGBkjh|ekkEKdV^2F>;@@hyb2 zaSh8?2x3U~r;w~KXQiAr2=S-;87C%j6FY(ul%O~(<~+d`SP4Z8ID~6I0z`pUn~U2N z_<|WJn>QQRVgWg%5EGJNX(eE?{RN3mH_H>MLNLUQpgQ9AazNmLEOqZfjRD@N|+7ii0>iQmUQ3Lr_Z)G0?%d0rWpz6V{u7XS#Aj- zaE9zh?8|EV+KY|clCQ6CXVZW7O|oYnZx}UM?X*Zsw;|OjTCviI>kZ|40{XDt{(OZ@Uzp~xMjbN8*PZmEHy?gCCN-#Hk zwA-Prq+pE?m6GV2Bs$+(n;#XhOw9{ZrofF z_YfHJ-mi6-v1tKcllm3Q4n5J@DeCsYm8JiLovHQEF`kTi-{qw8sBibG&85!|uNn3D z>{lOejvn^BVLB1>c=LGjB}Y)?4?W6?CsNP_q+<^Ft#Cu&=f=s88Y{I_)!$nHl1=x6 z*{JZJ^0^znG2|a{yfEpEd#jN-Fm_{FYCV6l@e5>sLB}AWUK|5*$F3XuWPE7#%=~>y zjKfOLN(}xK-HgNH(h`h#H)u)yR(~J{>+W9gW+&ejgAcPl!}o3jJ&t&D(lnc9&(-uQ zbfKq&o|gD(bAKHAm5v9<^`>7N`GPX+cKD9J^PXbKWQ^fyaijY-@0=dTHK9D7wv~>i z#PlQMcoh|=`0Je^xIW8|sa3#7@%VGCE;`d)xXOGO&0}#4oc6U~dRpx2g9QN#hlh$= zqAZ;qYZ$!pjX2REEDZ&-H`ygTvd^owq{EAq{pG^1;lH#mD&FTln`q|>yfl5UWUePb zZX{9^Ml~=f>p(-X6fBkUg4(;xH!YTkPKj{rY@ze%s9*+}zASnII8L8K9TcvC>_rSZ zTp6G?cqkzZFYm0Y>}05M|3LX)07Db#Rsdu%9_k&MA&y4b3Irsx)d^Jh2ml#dG){Kg z*Jh2(e1c=?OaD=jD|n--NQhdSl6E_mE^KA;z0_Ln2MzLAlGZ#xp4 zkM3R$zf9!yxA{Q!KmP!5_?LuLe$^fPE>nOc>hYHqzbyU@P4@P~`ON&XyrJzICgT00 zzZNQj@6v8}82npoV*Edfb^e=xe=P<0=M*FfjCdz7_@gp%0KaS>Tp;^X8@A=bdx8{d zWWM`F02|~M5ib)&8SuioGkWy>;r8`DhLpx<8o{d&hB6|aq z@P+b4y0zJrM2m_p>01}b{#@o2t${lhJ{!mo+KQCNUwHy z**W1BCI;10r_`ynJx12^*c-1nb!tf0^^v zfi>F)>+G)D@lcrV)q!Asp5D(3&cJ-yCP-h7be4;!m8V!rV0^SL>01}b{uD18$K+eT z;J!cQH%4bRh(1$suLz0b(vv1ku)l6&iTHtYqb!Gq*G4+e8g07gXa zZEuqF+}cbJ_c607E+uZ48DPzv@xIqEU3dPto>VT7y+HP73TeQ5*>>Ef(&80l=`g9{ zViCfWNg;zV(G{KIGImc<*1ez>%Pv)9K!w#POg90}fNl$5ut)=p!D*HZLoT8KO>!*g zR3L`mGvJ<0OZPZKv%`yw+y$~f3|T-Ozlq;b9T%dlP`xZTXQ!Zsg}Ll}UPXJQr76JE z5`L}$1ErTrurryIC$qquWX-D)TqQM4{7q!(BA0@)viERpbU!go|@c!#|b2oV38 zo_eJ^^L`o-zE-(@hYfu>c$GVR7jMC@&;8EdS6N>F6XUPX`4xEvzxe_e$o@@8u0I_E zh}j&B3@n7$nXV^tgA0qT(uk+M5(*A&CosDq#47?9!m25q4Rh^4GI%Ap6UcT(2^6qa_tBuUUuLA5sXfDnRyx&e)uuhMO>a zxZTxFW%Qs%<1H6=Z=Kf7pd9>>Yz*Lv#>9Q_n%l(lB~^0!ek(>Ya`dCTnKqBRc;dI0 zH~wR|J=phXwOFGK@Poee`r0){ER#7MS++1 zf-0#Ve_Aa{v+7{(O8)6)?LJCspyTw}<<eAxc3R1-cLkdN>K|>f`4E(eHzyq zE8QQAeV@iJkp0EC608>|^+t^~dBw$kcRSAAP*N9KJ^$1n51Y4oeccOOeK2Vx{=OwS zMMa~wba#GIELORhdwekcFljq=%hDB(cRGDgE5LbWYD6j*c=PJ2>P%&&R*VXUGcvZ` zL!v&{)T#0?QdWN?eMxlI&Xke_bLR8ikKIqTy_fcFaZug&Le&QKVmUXG# zlav!%bv>o^Y-g`~k;?)-YFn)>Yr10{bbxd7-XdO7?d)r5%}@2A#Ny;=+8cP1G!HM3 z{kv?~02v)7b!nDq{TLV5=Cvq2%2NtAZQeY zcIcb-!eK9mfxXG$4$0E(~O0;Y5GRT zVklA_IZ9u^)X_24>!wwga)M(>#I&p{4G5HwgTPsQn{UD}JlV!(=iLtt1O34k07FoQW3 zkFvOLB?Q?7vj3lU#q*C~mzzC1zdhN#g46YThPiEMp9>gdJdL@=!OE?NzHLkEo7%$= z0QrlyH^*V`RIjKNR#`8##c4{Ce$8LB-Botc5tbz}fmQ<9w}h;@%l&wO*inP8w%O}< z-NR&DZrjC9f4vO^V08?V$~d>q?u$0N#{;Puiik-MQs3Hxu0N|}hh_2D{dKqUY|ruR z)!N&y^6lc)+({stK=v(*TpK;}f?376H@VPPy8dFOln^+{DJ=W-0+Qr3w0`1_oDDW8*{#B(S0o=chQ6U&PpW1zqgvBe`gb-i`sGkzuwy=OobgMOw~+y8Dq+Sn|~_&<0wlR3}-Hh6E&L zNQGIm zYSB`WTKX4ORn?Z#WK&>L96gDEEgn{fBK5>2K?cfU)*^e!00LfmLE*-Y{mBsTG)0Sz z&kGn(bydB{D5HyLq5?$fcMAk8h~&;d0@?RZi+L8FPdnwTe@LHf!E+%4M|~a{t$9HY znDfi)NHBfrD}oaHJcNw7%F3T7@D7I`rTSCkUC4VwY1DIoD>_DTYmR$yodOnm%r{2{ zVCdunJL@@evikM@i(JazttT0|j>LC9Wxd9i1y5zy)X(9gURJzWdjKS{~c?YZD{fpGUnyt|HQxKy;8hjxXT+>e1G^m8mNv zkbU=%U7DBwqEKpmI(=={*7;sLJ_W8TR=H#LydeLnuAptDST1?pWBK#wai-KajjyBW zt^YXD>sQ-e-tGMow%hrdH`7YRzHM+N6&cZBri5_WtC(L$-eNedL6}~3+`*zaXlu4M zUG$yx_@K3V^@e*DoUGM0-*MEHUS4>&U1>Ky?XT(?sciV=fxJ?@wI7VN&H|QFySr*f zOEW-eQv*%~BEE2EBW0$Xn#v2vtd#dt+rHl|w8H zWb#b5Wb)=oo|r+?EHZL1iDln8lS|FMkllzoVF7x|*=gE>&Vq>G$z_pl&<0xC+XXJW zs6sstW9dZn^zg*3iOoXe_F539{OEGZv*99OBPvN4Tg}pj+<41Y=NiJZ1HoC)~kS|;K_6A>+ugPBDUjWaa zh3CaJ{Ic8j{OR~3UP{k2u|Im7*UZnStn}yXNB?4>r| z@#wSEd#u$;9#{gz^U7FE@s2IIIcnS;sUjt6!zlJsqDKnN69#AcHnmsaW7}~KN{QxC z0@>eRhB|E8+Sy%evk_h!!;LN77QkssSn|!0m*@-b| z0@>dU*-iByFAU?vRZY2D)bGm(h%2w~un2C8cZOq!&Rw~9yYD@;DXA+YbE>;GwrjU? z%dc+sk0g-f@6G8lJXA_)Vg9hNE4E>Wcim|?vwG^GqS+pYptY;oC!;s;Y**i9l*fAsQ-$WbL+ zKDHk$$V00#-|6B(HD+!O*}a_*lQXy?+z=8 zyNTP9n*J{AANiE*N-ayi_D8V05M^^p!Xt3nJzrVYH5wvj)x3x>^gn zV@9zD-SrX5^_n2kweeNP&W33n8)vjKE{h=yyA6S5W!|Q7^&-n&Ap17R5~2B1z!t!B zJqnR*V8}8BQCT7dBK0e8cu~L3dRnf10f>@fivSfIl#nU~idG`;j!|GtB03P1H46}> z2f(pHYs+s49C6_yl8?M&t#%3bxZG1ac3_8R@A1;cV>b>&H==(gx-KftweBp8U#Q4CC;7#09hVGmxuYG3xe@>g2vj5C%--@%|8cvHmxj;_-iSx>!J$`9D&35vM zTp;`2EC$~ECB|T%thfkH)eq$ql!%tkK$F8$j4R=G8Sa&XSG6%P#>(Y%?N#Y4Tf=ZZ z*D;b_;am>fUj$C@Ah2QilVVMX66RUNP~vd`vT7|N(i#w03=);R15Yuy5aw7TQyW%N z$X%0iWA!-xbPYMY5#be{;I8qgM#&v82Fx{H!s~my+k&+U$(8fD7$!%v;vdn^4Nv6qzxFfbpsP~ZH;%ESx7xiY z#!=a=rNss$uD6CRxJsQVHo>YU@QYeA*!%b0Tr^J_!Scu^3X;+>@$qUMbP0^!X^!0f z9E1e)?o}xe)P)mRk>Sjj=^>auv_T09PpqR25AiAy%qMQxWDH3^NbmyL_h#f8Jf4%Y zs?%Xw-T`VcmCgdWsXa+4D#uNk*?elOf}uY6XlgfirW+-n!%qCrX!i&oFiRjxE!R?- zFz&V8dn)FfERLgC?dj1Pt6zh}-Wf+{d^d^HZR&1qzs~FxOK3KCQ-_uQXgCXNHXJ=n z#lEVj9|g@0cCEf!m#}!}Ix#Z}Y4taQ%N(hx8?3*XTk=q~OaroI!_l@rg@F_?;gGs> zlXQL0dPurNX`^XaFmGPJbsru|-~ z&MA^&CDL%4`rSzA(4%TMmfdMQNbn0}-_Q8inOdVHK}Z1iF2GkW`GAc$;BRO@m_b~Vzh;TjeV867H; z*;>yv_sSarvbyzYL8XWL@Hp_)vh&QkD_m^|j(Hy`eIc7yS4`^c z$F2a}?!*v7@wiwLW_jn7k|Qi(p-YzViX=7qg2#%Po=XEni|b5T|Bjasv#op>$YBEm zc2+&AE4eGTdx{gbSaC4h-f+WtdCjh0tkgy200x01HCDNeUaU@cR@{SZlmskq}W(AxQa&vtS z%b!7I93;8&$yfMFnD0P>jydE-vkz}C=f#MB-(rm_^4MxG+f`tK1 zOafHDj=)G}*nu!@ZMZ1c)o>oM+n}m*6hxsAMKTx$2t=lbDaB`^kln3dfIxE+xH3l7 z!SlWwF5I9d)FHpUrn$TiL>WP+HK6YSr4C;p`>r4h{$c+&!tqdvMEF)U<+Um6g?{lT zd(&U7tEgU&@*ls&=loUAKV9lKL)Pml-n(u0;L+Ux03ZNKL_t*G0@<^XYx1uj>lnNk zy#5>o_)nJB7YG3GJp{gTfs}BF6}GHb1D@ynHrOqOycPJ%v*+Ho-xEAtlFKiUJsY`L z{#D2W>0o?dQny%?EC~BwFjL@kLLu= zZ|9z|&lRC%GPoa(gMl13Oldsb35l0k^@)V*CK!=}MSCyJlOb&ldU4o$@SKTfX!?@^ z059pZwJIIR&@X~xpzw$2fAPdY$Ja_hc`t|VGaC>F$eQJwS_--AxHEKVoiVjRf}wZglxsFskH-wkJPZtE9SH^xWOF zJ@fa8u?l8wPt$7trf*75Yp2}|9;A_(NRD9LPX#NSZAygZ)>rH8#-LbfgXBpfH0(&S z14%qAt1)9utf#;`I5!r1Df)SVY#|uPUGkMa^vo-t|HIxDwKl44VfTjp(zwOK2yhb0P=QT&?(zTV`$Y5TF^WCs&o~wW978Uo<0N5B@mVXnUj`?pg+PMA0R4YKGIvLB0;K;u_rwPMH) z59*1-X76ASKhuz>@}@+PgFqD3*dd~Vz~Ro-Y;hJ>eT+(#m_t;Bq=^NzcaBUE5HDe7)lbqvI!ITJZFL0m|2A;3d|+7WZ&nFJ%| zQT`hOPKSlx>@tUv%j9l=Wy2Xbh3p68epO!<+VcD=F)U8CZu-xV!bVkrwk+$hDhsaU zEE#`BbxmO^8hz}~J{t{TgWG#!vnm@6tt;qpvhDuW z<@7jJE)%+XJe8tY0#JALWD?fr#oUuNu91960k7BG zMak`0y`tlm{$(vaQuPIWxR18SZoB4B{hno4Ye9Xf_R!U|^L0~Q`h$k%dn@+S+#UqJ z%Aul7IO}t@OTFh;4^I%$y*p+*Et7efO~dG_*92U*XAbt&anE#cb4c$7TMqJTdT}`# zxX)kLwc&6%A9$BlyE<^M%7cqj$bKwbcHeygxww{gvpUG`W9QM$H-Bs^lDwzRQ=l53 z=bWG^tF6|4ZhNr7Uo1*HPp?u9 z&x;OHYi%x7B;uamzPj%Ev-W0oxwe-5iA^2z2?EZzgUtv@LMb)Hkjdgn5U6CL zHyt5@Vn~PPL;+##OpqNx!lB4V^a_K7rxS`-Qb`7(SrTD0b!0&~=jjy{)VP>&M{qXE zUjA?t=i+i^WW)-OB63s~;Y4Iw1ha^0u~JDdkSO9f6cr_0_RWsKm(13PG9{i7dD@ZX zi^xn@!h_C9rjwN7jtovA`(Z*BOGo@JX(zHogkJkviE@!8H<)0vFz4N305F1CqT>)% zHe#C{0FbSnNFc>5q=FC=d^USOlY&K3^G9^&^Bk_IcFAms#5^o)anmFXW{DF5QDh!4 zK^KZ=$WB1;;-T=1h%GD-uNMzUAR=`N!sc%Ukd`O;>Bv2y zLlQW`{&ilSi1^KJk!60OgNi3(;VJ$4Kp`8|{}R1b^aezsUjaz}IYxp8QvZY0`lnYm z@apa_-f{k4z`%b~hOP56xJf=xu{kG}h#63?5}vs>C(cnySSV=%cu^p*m@P^VA&X07 zG0n{p$Xj?9YcDJ=EB?%e(wL3I{7$buK~OH8fC|?Toy0g!A^YFbum1L3jRD|w6f>sH zD`$+Nm*f~yK#?hiITMsKmoZ!dCdk>YvLphrwk^wi_69fFh!-+L!4|C*RCLr7frMap z%26Pe6ner-3^|Q;+uQB^UGs6$eq<1+T=m+$rQPk0=F(z(?r@`Z z?0X5utFQ5JSoa63>s#$MXzf}zkh)XdqwVz?9B8iRQ`KJdT+41QDEX^wcjm(fb==C8 ziXHEp?C+v`Bk2#e!_Sc^hX1#Z86xL(<_d%(b`nGGG*SdokIM#J=eNcz@ z=~EI8vduxVyI$5T|I@J>RK937Ob0XmSn>O&Xr&e8YMe1hAclgWQR27Ly4ju zBq^vTZ9uTGC~U=>df~p_^nQl<74r)xE47`r^vuBA?U|HPVv@Cz!ejET=Y8JibteA2 zYtATl*qmsq5@JT5hUcDImyT9&|Deu$iI)ui2<*MOvNyPFjLaXchO;n~T9<e z`odNcfux@VPhM{w?Kfj=1qZ>)Ti@o8{WxJ;s=Td@0a6j_3ag*IaW!I^@~H@c(vc&{ z6m>xjsj)!Q6>&Ff&Xi%PznqE=>sP1_?}*Lo4%jJCQ;rJsPT_-=Y|GOYtpGDvrKfka zrta6)Fg>Bw2fYX=*c(S~mDm0HFpG5A&!RY;ZQ4JAD4>E?uH4EY`+@QN>KeVT_yZ8K z#ATag@QGDXCJR6q1C~Q2Oh5H~BiVh$L@ZGVkXl+6U>4R3kQA?iYzD;PbQoD(WhJrz z6zY&I$u7|iU>1~R!XRbUtt_T4y~^oKt03r~IhLNmT)CA)_5&Na3jg1ej_Cb${yoUD zqc43r-#*&k3Sv=i`dMc^8yx{BofHxRrU)m6||&D2FVFLJwG${ zikwM3H4T57>nby#CDa%3F~~?QaP?Aldm&~CZU3s{#;b0%msXzivo4abYi~%nm#qsu zM!K8Bi%^jeZHRrT)xGS6ms@}Sxym8?6EgPkaNXH}^Z|-ab{#8q(mgi& zs5A$s62$ll)2KK&ac1N%aI@;Ai!M_lW`ihwm-{k@N%?KX_7Mf#F&lO$`iu*GN&+Zd zw$+LGENP0!(wr}(fd`8q1_(nM%T-J&XaE9?L>OA=BoUPc9Vs$r$;wS*jP%U(Ggk=6 z_l1;6)HFAFU#o;dYIx!uDe~3O90Cg;VvrD35 z58C3Y{XNtUde~Krg*#rSNRMu7@r}Y;-st|k=9WYLdnfUV+DPhmg<8HRRIGx4+LbBrj6|&`L z$TrHQNBGlFGmVH?yVQ#B9KcK3^WF z1!J|l-BsKe>*qD9P@p^gwpRkd)+b1~*KGWw8Qe^k{l#&jjCz$N zR<~NO;crED&l_XEAE5!-`;D*cW>Oi4!+ucwK59?y8hRI33X8G0Ua**kzd{1p{%;0v z_{Q=6gy?Bc{kmr$DN!q_!Z1`hnB3#BXc%GRYpLEf+^wj#7n5oZ*`GErXg&u9p|ELh zPu9Ux_3WjIEVH7jvxVN8l`NAP4WotMHau%^UE@bz{eh>b5nr|s(D zL9Y>A+O4r4Gdwy(R?}UYTcJOsKZ0%}y z1!t@D)TT|LWjkEVRw5(z{F$yuoMlyqMigtXQ}{My;uV+19&=1VCXVHsj_$St(kZ;v zZL?#blZPvR)6-oY!=!^f!`9uMBl4OGZ_2tG_SI<)*`KlYtN+U2=qfX`CNivxwHzd& zC+SfK=yWqZgh`bNMkrkmG0an2PylekATdF`TBgqem^7~=0xzPTN<&<_kp3aEDpI;a zBw=EaOXE;7ae5jTs!19M(`+Gqs;3ik(vHh6ln#Sgoa;(OBB{#yE0C^vnRbj3IkQKk zVpG!4^m5l(OiYLjq6n*+S@#BV1EfSaoq_Oao>WBKOk}fvi_T-&pAfQiW^4V9Zo2x2 zdx9TOR!>yIld=zjKSKa~yF`CmMN1ssU(bK_LK8gi_|STgTrk`J!j-+J0GZ2wzvOelQSlS)+lcM#%q}rG1mYcy;K_VxFeyT-23A_D#a} z9U@50D+?I%9>tlHZ2GPx*)QU@YEtkQ0!Wq^xWXiJl*5^YE9TN+oM16%%0wm1OZ9W6 zb>YbSn#08NfPLCTTzUZ1E7@K;^Oge~C&8@lmVI(9h)U!hND2*=nxKp}tbdXZ!+hyr1MR*f9{RMi9;7UPqg?xoF^;`aT5?F`#fB}n?? zU$5}^u6i?O`ayX-ldZCIce30ZvN>d5F$V9&hifPNelI-WNf^ZEmAWs}IJmpMIqJhL zy{P*Q4zlg}aB(ygl|yVZzH@GMYaqs&VYN2P+VAnFSUXRW&7;H8rH7T%(}q^eBGm0l z168volk>@8R9Ca@;Bh7a;DeQgP#a;I(5{zgwzH#bEN}Z4TL2 zylmTN-^2yWf_K}84>M~dTAwA|Hff|Dg6^ojKJec}e%+*z*FE$foaveugwy@)X{7PT z`aB#P{pL6QbW|P;L9J~xm+7$A6}~&L6s_sjtwZfW*A?4RPrO_l4Fl!cbSs0=&QPP? zm*JunFQ*M$aZUcgwRWrPO?!RT4!y?hOf~ahHizts#^7%(cq{UQbqbj^T&w-UCk2== z1;Q)v36+3ptsplMUx##Yv1gkF)U1g?-K^0(E$+AiLr*K8e8QxFOX)3W}_-u1pFuB_qAzs& zwMb*h!L&DLng=BI9hFlMMI{e*iExmwV>)?t8ml*9Zl`;E+cok)}!n9 zdN;bi%gA59EUI&B&@(S~C)X`}`e9Qw9mkm`z1vbc`Y?}*Gx_9kaa{);t={Z7r`1(B zzjDvd=abJ#KxO3kD|~s^x6(#)XkQm6pW{_E?%4I_Nq5y~Zh8-Tu&Oz;`OW2WWy)@0 zz8XyG^X{pAT{vqd)<8z`m*#-RHaZWQd+SZ=wej`x!YsNa0G0Y>W!3Pyok{(qdD2=N z=gzcP++NSR9hXcY+4}MOp02Ks*7d>7sstow=CoF2+>_}_o0z)#deQ>2c^NtzZ#Zc| zd3v{V=a1wnu1R{&hNE9kNN3tyIZbzWm)0I^=l;YwQvg)&TlM;hGfr+-*5s;pc3#X= z_AQqZ8t!4SRQ5)aexqzAqni#JhDy_I)ST=ly+QxOv{Gz7cDv2lW;?tM-5&3fSR)-S z$4%WSUU_Ei;^_z}kd0+e9 zg?sKetL|drL|t)mINW#*=X_Ebx^_p-V7R`q zhK}~2JE`L=3zs|haeka6^|SbXBqK;=#ZYz`lvfL9bITE`qDYiQ}#{Avfm|xmqBM`u{Z=@=2iGffUY|5KBZj`McHaD6j zFCL@OiN1+c>(iG3ny2rhoo?uOQ_``TMt0$Kn>d2;g33XXSM z+zPc6PrpW1!l(K|_pLz3y8YQtI$Agq0x28^i?px}%6$vB5+#jwZ0ttmR8KTGJ=b@; zQmR)S$;L*t5_hX{qo9O~bz}_5K-M>RJ_q9I&h9RBtXLLFV5JJHJCJ~5mwvb1)K)c~ zJmj&Iq~yX2cSCNyyWw=boZ9`d)t}n5HO01J2PlF#CihU%k4xU1vn&G zPh?WyZ8F6ITguTPw?nQ?S;VZBy7DnqCXs$O2Eft!OSo6oqZJY`M-1K$myQ3s<@@%l zOVGdUA4Pgz%Dp|e{QO=t_=jJwOCAsQ48IRy{ra=^UyaxMHze$Pr|h4P7+ALSGGoi|AFv!`^x<1Z@7OXxfj%P(04)W* zN^;cq#uDu(%z-_9l zWA9c0p#0TrRBGK@yVh8&yZyU)u0@om>>Gky{ngQnlpU7Rdsn%VX0=*V=}((6@VyJ~ z=Bru170=X#CnHX7^4-DAK0RulFF${5uJV-4Q}!)c39roU;fCjXf#0k2$JVHzSwR_z z=D3oYw&&UF0q<&ngDI~$gUq!DuAWNz`8Kb#%2W0YxnHj)-L!a$xy1RBPpPW3vV`nP zbc2$}=!IUtlLqpXeXErHkzt#^@|6Am?qvu6_P^t#P^Pu2{$XHBE2Ex4 z0q~hK3nIG+LW!30!SBp$@QoRcqL4@eHB+HQLAVL-zA>;t=OWP#?p$K-O2FTE& zMXR?Zb`t56g~_|f#vJ4)j4?So*~rL)#*)KG$}Pb@c1vT)r($uJ$^T*R>RJ>>)^Ji# zH)ASfpoMcKm?b7nPD~<+Nf^>%C-EaBqG`kkiKq>NI-+1Vebas6axUjI>|fbmcv8`x zkKHq8Ic(qbjNTZkUMt%DSXRCDJkOiOVx|@D$fdp6Go$mgj=xg)ybBb0F=uO$d|1{i zWFC-xpvdL~Nf>qMJk|UQuODmI7C}A;JVW-)WDJ7&`c%VKv-rR-wPfUV0Rh@0nvmPl zpa)mkwdKTn#^CO_w-S_J^L(LL~Sr<+3-BC8@T32!X) z`|!HxtVYWQrOb2jBXGtyvXg}jEoCumjh$$bPPUrEPktA~Sty%>hsVQaw3g9$>uI4lcYzfR+{La{ z)JvAAIa@8^wcnfftI5D=xZz=G!df;hs@7K7h@-ikx}i4&p_<0EqAra?d!v=m(XQj8 z#_M;S?}iU^fh;CV&6lTjv6LH=SLAAKz=PsQqX%M58}Q%^*|&1~)p}xH7Pg&>#iyaO zl6)~ALigAxy;FMqSo-Mk#AdLtMrPwFoev+=O7ijWY3j{_N3EJ%ZF*Dt?t|9NAl|p^ zQDBA>-4x(>9S+zDX!SznVd~B!#{sCc?8S|FFzAY7(V365ZPyYze$tAOwHX z&`VMkt_kw8Npnq-et#63un-g|z-6;ic1Z|j#34{d>jD(YTp8e4fw+X1W!+QtY(gVA zY^crY+18AM;D*3)VhFR84@UAi`gYjk%%HMk+|fD_t`C)G|0w z6H-l%ZsJkbH)Ix8)?oor;Jtx$GGzg)aWFpu5I1j@DDe|AxyoPv9_BS*A^F94i##!6 zuJcRWaeWPsnm_zcKE)ozu^Bv1N?GE_ra8$KbJgQ#8EtM;2#++I>!Wt?DC8wiFQdp~ z0H$Oy=w+}h_8Oi>$osJxWBLmK01k~wL_t)cJ#M1F>4wKQjYp)8MH%v{kXY;ca^sVF z5)v;4be})w3;X8`**5@L{qXPqWdr#CoDo3qCnDjuGuMBwG5a^M^0V`Ceoj>a@du#c zq8y0Cglf>LFi3|dku}U)QC@`mNYKWJ#vE`Gw-Qr^wXuyWL!#q|5UsPA6AhX{?aQ<0 z%Neq-qy0+0!7OO+3R0CaSiTu`Yvd+d_`%C4`?VzxPwTkZj_J?=QdMPn+b?H*Ev%kt z$Lm%X>?bU~q&r5QHUsrJ->w5?Q80#4Ox63$8DeM1zDCC2v%>bYBU_GwhmA3dYsN@O zKF+3qK%>nR=t?8-oX`*Ln)vkTzWQqr_$1Kj?NbuSx1s2C^!v_pFuAK~4N=9Jd42a= zC)syaO5Yo}^WoY$lV{G5eVtkMS^wI6g=tXIsykh3T58%eJ&Z{L9Fhvqr8IE0ntbTS zjl*@HNiVKmH*PO$gf@u}jxrh^}I?=8~91oPJTp@YzPT?btYnu#9 zVCRd=GAn_knJ`;0)Qn4O!HV(Vqz>5P{s)2ydmx&;7Zh5O#sb z7Dl+%B1AZaMCPVK*+b2fPvGbW^g=qp&PR}H)7d61I%~wvkbMQnR_f@FDWlKHsP7=` zd$lybTiyfa9#;f7Ck)PzeVv8Dm(wqhmy3;PGS6udG(0*8=E?n+dL1OW&OJGgT{z(n zPX`Lv;Pd(=_V~iwZ-6?oZeTVp$W#8gT^3H2p_8E-{4yAMGVMHZ72$Ebulx$nj~^Pc z5EU8nueM(o2kErNWbC^`Wb0$?f}=W zNvo(y6Kjf80a3;zTLD7#hGL;mKtgn0Rr~~yMX=-B#OZYF`;g_oFu8hceREOq)YW75 zbI$RkP|IfBqT4ZMyPjY17exYTbH|!{+l@Q$bV&Mx>8#h@9-LUTR_&Qv><@!1))JQ% zg_v7^l=P*jnNN{=MJx6cap}2RkpbB)4*Nz4ALg!c>x6;phFop4i) zZpJoJDGfW@_^h7z@ga+<_gizm67|1Gav2-qg)ciIxH+tc#=`_xfF+A!P~wF`ttiiA zc{1&rC0)Pm%X@kJ=x^d%h5=gddG9Hc55d0rFc{vJ=#ZJrYO*6cR@0W&-+J5RbQiwe@i;n0nEI>B>w&Jg}erhbuy;Uh)jaS`K{J|1hNf^i% z*JphG@Q^K;7HbTR`H^q`eT8kbK22aqii!t+*d4Ilyk-{xx}2_5t3heuELL6aKkS`R zZyL)L$9F~@T`BC)wVIWhD_yQYUD(xjmj!tYNFiJTcozf+u-4!R=t3lwvIRoqDxmGZV~NLp5|mIEtdruJ8b)mUGRQ=}sf`#X(xNI*)CosHYg zSifsTEVGyZNKDW2UBfO(0@C6IBeo3$m@2l?=~yAH>D(B-vR%y*3# zlLlliSN5p9uZ#=uc28e-XDJLN5dB`z}P@}A_-B5{TcOyNy3_0zfRkt#JUZIGs}_vILIRGv=4Jj z5&WMd*VaMcJcLrA8BG?q{$gjBRwHL+HW7l>fT<;$n?!)kdN;&L;mCfRyeuLl2wwN+ zUtjsK9-BR#w=(;^Y`PcQ?)LHkfjK>O%0-VNJI<6ivK-lucVYWBzPb8fu#8FPmvBRC zWwNkm>6L0Z;Evj1geG13s9>LlQKX!)dD$yR_7m0^r2j^KpyKUob^G})HuMv%>kSzC z6E=_y(Nzu#NA}Yp3j+D82}GCv&T~&3*-weAl={cILz=v)^F{=oohQJ#qs2Lg3yImc z<%)05Ac*h79NCZ9Nb{W>q~HIdWldxPWP)X2#N=QA0}NHB)E3-f^N(cgZ3SfsQU?hT z8Tb>Ir9cdZSt=|%p*~23{;VVciB4!epU9t_#D*Qo9Ab{_2kiZt)gTaOhf2!WXjsv* zIn6C=wi9+&HF-O`lk>DjWUC!r=o{IrlD5pWXf-s`g#wPwyJfu86ctzLx~^Kx60{C99NdmU;w)>zj>cCWvrk zVILhB1GChA% fvL6iDyPd2A@`tmnh3+*T#FND*Ij54G945RYvoJ6pC(E_crd2wg zwQBoz;XyK2-3-f~0R_^pq|0M)oD~l{sYOR0ta9+JII|&kmMAt=;mZ0Db6FM#A*+7L6L(Rto*A^T>5qHdGPz7Qb z2OJ!p05y*U5VYy4s0oS@3z&%h7z+nQ)(jqt1eyvEK+IFLC{O`#f_#_d$X?^ZR*V?Q zSuFg%3tJeo{QVDM%=)>Qwjk@9rN4oPkoaA+Y#1#-fGr7Y2XK}ipq>nO1enAT{d@p; z*0crjdIZDz2~2a4XT~_P*C~tX9QMlb#ay}!*)xV7{ryjmUSE@;3pZxyjZok_tLKxm z0G%lgGEZ=p`SLpt-_(MkEAqV;K;}$yWa*c_UieMuZnSb4vUKnB8?t`&@BjXm=@r5W zl7e8p$xhe^P`E*0P?`v5s{n;HHp(M_2~+2SL5SD4ViBZm00IQ{mtX>!sxZ}s~TMI&rA>jGR+RA)XCe(ZyU1ZLP)^oWynhI zG`P`PBYQX1kwB|gv*sl%z*Jq!Wp)tO>!a-BQ8H)DH?e>_^L9$NX9mkcBZDvDCBi5>~sTEG-c6DCybDET%UJ|7L| z2Z@Ot4#Y@QC00%WEhONCs)%YVja5_FG@z++W^Wg=v%P&m@*d0YAdAGzP+gRWkL=G9 z-$wAANKWWinj<^m{_R$a!-F3S!h0a8aXP+KC%yH z830fvs}4P4Y@iCtw27@+Q|(EwTq`R>8lRy_Mbh>CNWz=ub8%g8!b9Uh(t&~;kA~{i zLi4C;wwiroNgoOFJyba;^>c}#JFINcj|B=9M#=c1^0+b|7raW1l|y2CdKj2`dI}yT zJICYALe71fhr~K6MW3d225NB;i5q4m2P{tIZAx-Uv&B|32QJYufY^>W#`<9Jl}3qG zx>>hW1uG3L0|h%YydCQ$h zTUn{-QM~VXaHL1eMq+Q3(EV?AXeMk|UwTGjWGK4FatUZ>)`@J6A`yAnQD&WR+Oldg zJ;GKk({z_{+n}F|Sy=ak5vTH2X@X+_AZZd%(^14rqhFJB)`cxMjx0x(8%LHS%Z($;jU&sE<;Ica$a3Sza%4HOpE^JvLPM*Jb{_5nP_s=2AjoW97?C#6cljkQdUY(x$T`C+|Zl67}r>7?`PQRzf z-us3l%k8IdFgShw;@Pv)SI%ThcmJpj!%zUmm4HWspy&Y2;;9r0 zA{pFLf=xP^v|y7he-{siF5N;PSMw6G{E0|hlF0h55C_SKFpIPc+NGx^RBkc zvJml=-FfGm&bc$kFNjE1#iy?JO>?_ZF`8{70E(GvPu;XMR9*&3-mhK z|IcCok_AW>Bula&S%73gvLp*($%e-2`?bqJpOe+m%YJ*b0m(u}vNm6Tk5*-(NfsiK z4F=Wft9lK8uA2=>7JW~47?tMnxmqR@L$VN(tTt)BOy(pD8G}Jk`. +**Pro tip:** Use `__init__` to hold references (pointers) to other `Object`s or immutable state only. That is because a charm is reinitialised on every event. See [Talking to a workload: Control flow from A to Z](). ``` @@ -228,7 +228,7 @@ Finally, define the `pebble_layer` function as below. The `command` variable re def _pebble_layer(self) -> ops.pebble.Layer: """A Pebble layer for the FastAPI demo services.""" command = ' '.join( - {ref}` + [ 'uvicorn', 'api_demo_server.app:app', '--host=0.0.0.0', diff --git a/docs/tutorial/from-zero-to-hero-write-your-first-kubernetes-charm/integrate-your-charm-with-postgresql.md b/docs/tutorial/from-zero-to-hero-write-your-first-kubernetes-charm/integrate-your-charm-with-postgresql.md index b83f70ffe..e6e711488 100644 --- a/docs/tutorial/from-zero-to-hero-write-your-first-kubernetes-charm/integrate-your-charm-with-postgresql.md +++ b/docs/tutorial/from-zero-to-hero-write-your-first-kubernetes-charm/integrate-your-charm-with-postgresql.md @@ -245,14 +245,15 @@ def _on_database_created(self, event: DatabaseCreatedEvent) -> None: The diagram below illustrates the workflow for the case where the database integration exists and for the case where it does not: -![Integrate your charm with PostgreSQL](integrate_your_charm_with_postgresql.png) +![Integrate your charm with PostgreSQL](../../resources/integrate_your_charm_with_postgresql.png) ## Update the unit status to reflect the integration state Now that the charm is getting more complex, there are many more cases where the unit status needs to be set. It's often convenient to do this in a more declarative fashion, which is where the collect-status event can be used. -> Read more: {ref}`Events > Collect App Status and Collect Unit Status ` + +> Read more: [Events > Collect App Status and Collect Unit Status](https://juju.is/docs/sdk/events-collect-app-status-and-collect-unit-status) In your charm's `__init__` add a new observer: diff --git a/docs/tutorial/from-zero-to-hero-write-your-first-kubernetes-charm/observe-your-charm-with-cos-lite.md b/docs/tutorial/from-zero-to-hero-write-your-first-kubernetes-charm/observe-your-charm-with-cos-lite.md index b0b8bb9f3..0dd3c8f1f 100644 --- a/docs/tutorial/from-zero-to-hero-write-your-first-kubernetes-charm/observe-your-charm-with-cos-lite.md +++ b/docs/tutorial/from-zero-to-hero-write-your-first-kubernetes-charm/observe-your-charm-with-cos-lite.md @@ -91,7 +91,7 @@ Now, in your charm's `__init__` method, initialise the `MetricsEndpointProvider` self._prometheus_scraping = MetricsEndpointProvider( self, relation_name="metrics-endpoint", - jobs={ref}`{"static_configs": [{"targets": [f"*:{self.config['server-port']}"]}]}], + jobs=[{"static_configs": [{"targets": [f"*:{self.config['server-port']}"]}]}], refresh_event=self.on.config_changed, ) ``` @@ -449,7 +449,7 @@ In a while you should see the following data appearing on the dashboard: 3. Logs from your application that were collected by Loki and forwarded to Grafana. Here you can see some INFO level logs and ERROR logs with traceback from Python when you were calling the `/error` path. -![Observe your charm with COS Lite](observe_your_charm_with_cos_lite.png) +![Observe your charm with COS Lite](../../resources/observe_your_charm_with_cos_lite.png) ```{important} diff --git a/docs/tutorial/from-zero-to-hero-write-your-first-kubernetes-charm/preserve-your-charms-data.md b/docs/tutorial/from-zero-to-hero-write-your-first-kubernetes-charm/preserve-your-charms-data.md index 138f2a90a..6d2f14d2b 100644 --- a/docs/tutorial/from-zero-to-hero-write-your-first-kubernetes-charm/preserve-your-charms-data.md +++ b/docs/tutorial/from-zero-to-hero-write-your-first-kubernetes-charm/preserve-your-charms-data.md @@ -28,7 +28,7 @@ There are a few strategies you can adopt here: First, you can use an Ops construct called `Stored State`. With this strategy you can store data on the local unit (at least, so long as your `main` function doesn't set `use_juju_for_storage` to `True`). However, if your Kubernetes pod dies, your unit also dies, and thus also the data. For this reason this strategy is generally not recommended. -> Read more: {ref}`StoredState <7433md>`, {ref}`StoredState: Uses, Limitations ` +> Read more: [StoredState](), {ref}`StoredState: Uses, Limitations ` Second, you can make use of the Juju notion of 'peer relations' and 'data bags' and set up a peer relation data bag. This will help you store the information in the Juju's database backend. @@ -49,7 +49,7 @@ peers: fastapi-peer: interface: fastapi_demo_peers ``` -> Read more: {ref}`File ‘charmcraft.yaml’ <7433md>` +> Read more: [File ‘charmcraft.yaml`]() ## Set and get data from the peer relation databag diff --git a/docs/tutorial/from-zero-to-hero-write-your-first-kubernetes-charm/publish-your-charm-on-charmhub.md b/docs/tutorial/from-zero-to-hero-write-your-first-kubernetes-charm/publish-your-charm-on-charmhub.md index a08f97613..74f76f35e 100644 --- a/docs/tutorial/from-zero-to-hero-write-your-first-kubernetes-charm/publish-your-charm-on-charmhub.md +++ b/docs/tutorial/from-zero-to-hero-write-your-first-kubernetes-charm/publish-your-charm-on-charmhub.md @@ -82,7 +82,7 @@ echo "Random 8-digit hash: $random_hash" ``` ```{important} -Naming your charm is usually less random than that; see {ref}`Charm naming guidelines `. However, here we are in a tutorial setting, so you just need to make sure to pick a unique name, any name. +Naming your charm is usually less random than that; see [Charm naming guidelines](). However, here we are in a tutorial setting, so you just need to make sure to pick a unique name, any name. ``` @@ -161,7 +161,7 @@ Release your charm as below. ```{important} **Do not worry:**
-While releasing a charm to Charmhub gives it a public URL, the charm will not appear in the Charmhub search results until it has passed formal review -- see {ref}`Requirements for public listing <12281md>`. +While releasing a charm to Charmhub gives it a public URL, the charm will not appear in the Charmhub search results until it has passed formal review -- see [Requirements for public listing](). ``` diff --git a/docs/tutorial/write-your-first-machine-charm.md b/docs/tutorial/write-your-first-machine-charm.md index d3e8da698..1bd48dda5 100644 --- a/docs/tutorial/write-your-first-machine-charm.md +++ b/docs/tutorial/write-your-first-machine-charm.md @@ -286,11 +286,11 @@ config: Then, in the `src/charm.py` file, update the `_on_install` function to make use of the new configuration option, as below: ```python -def _on_install(self, event: ops.ConfigChangedEvent): +def _on_install(self, event: ops.InstallEvent): """Handle install event.""" self.unit.status = ops.MaintenanceStatus("Installing microsample snap") channel = self.config.get('channel') - if channel in {ref}`'beta', 'edge', 'candidate', 'stable']: + if channel in ('beta', 'edge', 'candidate', 'stable'): os.system(f"snap install microsample --{channel}") self.unit.status = ops.ActiveStatus("Ready") else: @@ -345,7 +345,7 @@ Next, in the body of the charm definition, define the event handler, as below: ```python def _on_config_changed(self, event: ops.ConfigChangedEvent): channel = self.config.get('channel') - if channel in {ref}`'beta', 'edge', 'candidate', 'stable']: + if channel in ('beta', 'edge', 'candidate', 'stable'): os.system(f"snap refresh microsample --{channel}") self.unit.status = ops.ActiveStatus("Ready at '%s'" % channel) else: @@ -437,7 +437,7 @@ class MicrosampleVmCharm(ops.CharmBase): """Handle install event.""" self.unit.status = ops.MaintenanceStatus("Installing microsample snap") channel = self.config.get('channel') - if channel in {ref}`'beta', 'edge', 'candidate', 'stable']: + if channel in ('beta', 'edge', 'candidate', 'stable'): os.system(f"snap install microsample --{channel}") self.unit.status = ops.ActiveStatus("Ready") else: @@ -445,7 +445,7 @@ class MicrosampleVmCharm(ops.CharmBase): def _on_config_changed(self, event: ops.ConfigChangedEvent): channel = self.config.get('channel') - if channel in ['beta', 'edge', 'candidate', 'stable']: + if channel in ('beta', 'edge', 'candidate', 'stable'): os.system(f"snap refresh microsample --{channel}") workload_version = self._getWorkloadVersion() self.unit.set_workload_version(workload_version) @@ -505,8 +505,6 @@ microsample/0* active idle 1 10.122.219.101 Ready at 'beta' Machine State Address Inst id Base AZ Message 1 started 10.122.219.101 juju-f25b73-1 ubuntu@22.04 Running - - ``` Congratulations, your charm user can view the version of the workload deployed from your charm! @@ -523,7 +521,7 @@ By the end of this tutorial you will have built a machine charm and evolved it i | If you are wondering... | visit... | |-------------------------|----------------------| -| "How do I...?" | {ref}`how-to-guides` | +| "How do I...?" | {ref}`howto-guides` | | "What is...?" | {ref}`reference` | | "Why...?", "So what?" | {ref}`explanation` | From 14e0980a2d4763aae9e7aed529ac51d386a3df65 Mon Sep 17 00:00:00 2001 From: Tony Meyer Date: Wed, 4 Dec 2024 10:45:08 +1300 Subject: [PATCH 05/16] Fix spelling errors. --- docs/explanation/storedstate-uses-limitations.md | 4 ++-- docs/howto/manage-interfaces.md | 2 +- docs/howto/run-workloads-with-a-charm-kubernetes.md | 2 +- docs/howto/write-unit-tests-for-a-charm.md | 2 +- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/docs/explanation/storedstate-uses-limitations.md b/docs/explanation/storedstate-uses-limitations.md index a81ca4093..5cbf52046 100644 --- a/docs/explanation/storedstate-uses-limitations.md +++ b/docs/explanation/storedstate-uses-limitations.md @@ -84,7 +84,7 @@ def _on_config_changed(self, event): One common scenario where charm authors get tempted to use `StoredState`, when a no-op would be better, is to use `StoredState` to cache information from the Juju model. The Operator Framework already caches information about relations, unit and application names, etc. It reads and loads the charm's config into memory during each hook execution. Authors can simply fetch model and config information as needed, trusting that the Operator Framework is avoiding extra work where it can, and doing extra work to avoid cache coherency issues where it must. -Another temptation is to track the occurence of certain events like [`pebble-ready`](https://juju.is/docs/sdk/events#heading--pebble-ready). This is dangerous. The emission of a `pebble-ready` event means that Pebble was up and running when the hook was invoked, but makes no guarantees about the future. Pebble may not remain running -- see the note about the Kubernetes scheduler above -- meaning your `StoredState` contains an invalid cache value which will likely lead to bugs. In cases where charm authors want to perform an action if and only if the workload container is up and running, they should guard against Pebble issues by catching `ops.pebble.ConnectionError`: +Another temptation is to track the occurrence of certain events like [`pebble-ready`](https://juju.is/docs/sdk/events#heading--pebble-ready). This is dangerous. The emission of a `pebble-ready` event means that Pebble was up and running when the hook was invoked, but makes no guarantees about the future. Pebble may not remain running -- see the note about the Kubernetes scheduler above -- meaning your `StoredState` contains an invalid cache value which will likely lead to bugs. In cases where charm authors want to perform an action if and only if the workload container is up and running, they should guard against Pebble issues by catching `ops.pebble.ConnectionError`: ```python def some_event_handler(event): @@ -101,4 +101,4 @@ In the other cases where state is needed, authors ideally want to relate a charm In the cases where it is important to share some lightweight configuration data between units of an application, charm author's should look into [peer relations](https://juju.is/docs/sdk/integration#heading--peer-integrations). And in the cases where data must be written to a container's local file system (Canonical's Kubeflow bundle, for example, must do this, because the sheer number of services mean that we run into limitations on attached storage in the underlying cloud), authors should do so mindfully, with an understanding of the pitfalls involved. -In sum: use state mindfully, with well chosen tools, only when necessary. \ No newline at end of file +In sum: use state mindfully, with well chosen tools, only when necessary. diff --git a/docs/howto/manage-interfaces.md b/docs/howto/manage-interfaces.md index 1441b35e1..588ddadae 100644 --- a/docs/howto/manage-interfaces.md +++ b/docs/howto/manage-interfaces.md @@ -462,7 +462,7 @@ Essentially, you need to make it so that the charm runtime 'thinks' that everyth This may mean mocking the presence and connectivity of a container, system calls, substrate API calls, and more. If you have scenario or unittests in your codebase, you most likely already have all the necessary patches scattered around and it's a matter of collecting them. -Remember that if you run your tests using `run_matrix.py` locally, in your troubleshooting you need to point `interface.yaml` to the branch where you commited your changes as `run_matrix` fetches the charm repositories in order to run the charms: +Remember that if you run your tests using `run_matrix.py` locally, in your troubleshooting you need to point `interface.yaml` to the branch where you committed your changes as `run_matrix` fetches the charm repositories in order to run the charms: ```text requirers: diff --git a/docs/howto/run-workloads-with-a-charm-kubernetes.md b/docs/howto/run-workloads-with-a-charm-kubernetes.md index d44af0329..e89833c41 100644 --- a/docs/howto/run-workloads-with-a-charm-kubernetes.md +++ b/docs/howto/run-workloads-with-a-charm-kubernetes.md @@ -849,7 +849,7 @@ pg_dump mydb >/tmp/mydb.sql The first argument to `pebble notify` is the key, which must be in the format `/`. The caller can optionally provide map data arguments in `=` format; this example shows a single data argument named `path`. -The `pebble notify` command has an optional `--repeat-after` flag, which tells Pebble to only allow the notice to repeat after the specified duration (the default is to repeat for every occurrence). If the caller says `--repeat-after=1h`, Pebble will prevent the notice with the same type and key from repeating within an hour -- useful to avoid the charm waking up too often when a notice occcurs frequently. +The `pebble notify` command has an optional `--repeat-after` flag, which tells Pebble to only allow the notice to repeat after the specified duration (the default is to repeat for every occurrence). If the caller says `--repeat-after=1h`, Pebble will prevent the notice with the same type and key from repeating within an hour -- useful to avoid the charm waking up too often when a notice occurs frequently. > See more: [GitHub | Pebble > Notices > `pebble notify`](https://github.com/canonical/pebble#notices) diff --git a/docs/howto/write-unit-tests-for-a-charm.md b/docs/howto/write-unit-tests-for-a-charm.md index 02b8cb54b..d8edb8285 100644 --- a/docs/howto/write-unit-tests-for-a-charm.md +++ b/docs/howto/write-unit-tests-for-a-charm.md @@ -23,7 +23,7 @@ def harness(): # Set a name for the testing model created by Harness (optional). # Cannot be called after harness.begin() harness.set_model_name("testing") - # Instanciate an instance of the charm (harness.charm) + # Instantiate an instance of the charm (harness.charm) harness.begin() yield harness # Run Harness' cleanup method on teardown From 47c6590e4be58c8153dc5adadddc0c60c441f8d5 Mon Sep 17 00:00:00 2001 From: Tony Meyer Date: Wed, 4 Dec 2024 10:58:39 +1300 Subject: [PATCH 06/16] More precisely (temporarily) disable the warning that is causing issues. --- .readthedocs.yaml | 3 +-- docs/custom_conf.py | 6 +++++ docs/ops.rst | 64 --------------------------------------------- tox.ini | 4 +-- 4 files changed, 8 insertions(+), 69 deletions(-) delete mode 100644 docs/ops.rst diff --git a/.readthedocs.yaml b/.readthedocs.yaml index e8a02993d..2edd6f79d 100644 --- a/.readthedocs.yaml +++ b/.readthedocs.yaml @@ -14,5 +14,4 @@ build: sphinx: configuration: docs/conf.py -# Temporarily remove -W (error on warnings) so we can preview results -# fail_on_warning: true + fail_on_warning: true diff --git a/docs/custom_conf.py b/docs/custom_conf.py index 2880c6cae..579f7e30a 100644 --- a/docs/custom_conf.py +++ b/docs/custom_conf.py @@ -25,6 +25,12 @@ from sphinx.util.docutils import SphinxDirective +# FIXME: temporarily disable missing xrefs in this branch, until the missing +# links to Juju and Charmcraft are added. +suppress_warnings = [ + 'myst.xref_missing', +] + sys.path.insert(0, str(pathlib.Path(__file__).parent.parent)) # Pull in fix from https://github.com/sphinx-doc/sphinx/pull/11222/files to fix diff --git a/docs/ops.rst b/docs/ops.rst deleted file mode 100644 index 29d9c6530..000000000 --- a/docs/ops.rst +++ /dev/null @@ -1,64 +0,0 @@ -API reference -============= - -The `ops` library is a Python framework for writing and testing Juju charms. - - See more: `Charm SDK documentation `_ - -The library (`available on PyPI`_) provides: - -- :ref:`ops_module`, the API to respond to Juju events and manage the application; -- :ref:`ops_main_entry_point`, used to initialise and run your charm; -- :doc:`ops.pebble <_ops_pebble>`, the Pebble client, a low-level API for Kubernetes containers; -- the APIs for unit testing charms in a simulated environment: - - - :doc:`State-transition testing <_ops_testing>`. This is the - recommended approach (it was previously known as 'Scenario'). - - :doc:`Harness <_ops_testing_harness>`. This is a deprecated framework, and has issues, - particularly with resetting the charm state between Juju events. - -You can structure your charm however you like, but with the `ops` library, you -get a framework that promotes consistency and readability by following best -practices. It also helps you organise your code better by separating different -aspects of the charm, such as managing the application's state, handling -integrations with other services, and making the charm easier to test. - -.. _available on PyPI: https://pypi.org/project/ops/ - -.. toctree:: - :hidden: - :maxdepth: 2 - - self - _ops_pebble - _ops_testing - _ops_testing_harness - -.. _ops_module: - -ops ---- - -.. automodule:: ops - :exclude-members: main - -.. _ops_main_entry_point: - -ops.main entry point --------------------- - -The main entry point to initialise and run your charm. - -.. autofunction:: ops.main - -legacy main module ------------------- - -.. automodule:: ops.main - :noindex: - - -Indices -======= - -* :ref:`genindex` diff --git a/tox.ini b/tox.ini index 43442c04e..ef9ee7f90 100644 --- a/tox.ini +++ b/tox.ini @@ -47,9 +47,7 @@ deps = pip-tools commands_pre = pip-sync {toxinidir}/docs/requirements.txt commands = -# Temporarily remove -W (error on warnings) so we can preview results -# sphinx-build -W --keep-going docs/ docs/_build/html - sphinx-build --keep-going docs/ docs/_build/html + sphinx-build -W --keep-going docs/ docs/_build/html [testenv:docs-live] description = Live development: build the Sphinx docs with autoreloading enabled From ac162a9757ada39fdee3c86f09cfbdd84f62ed4d Mon Sep 17 00:00:00 2001 From: Tony Meyer Date: Wed, 4 Dec 2024 10:58:57 +1300 Subject: [PATCH 07/16] Fix the API reference and remove the unused page. --- docs/reference/api/index.md | 7 +++---- docs/reference/api/ops-testing-harness.rst | 2 +- docs/reference/api/ops-testing.rst | 2 +- 3 files changed, 5 insertions(+), 6 deletions(-) diff --git a/docs/reference/api/index.md b/docs/reference/api/index.md index 4ec53d403..2a5c77f38 100644 --- a/docs/reference/api/index.md +++ b/docs/reference/api/index.md @@ -6,8 +6,7 @@ ops-main-entrypoint ops -ops_pebble -ops_testing -ops_testing_harness +pebble +ops-testing +ops-testing-harness ``` - diff --git a/docs/reference/api/ops-testing-harness.rst b/docs/reference/api/ops-testing-harness.rst index b82c0c6d5..4ee02a5ec 100644 --- a/docs/reference/api/ops-testing-harness.rst +++ b/docs/reference/api/ops-testing-harness.rst @@ -1,7 +1,7 @@ .. _ops_testing_harness: `ops.testing.Harness` (legacy unit testing) -============================= +=========================================== .. deprecated:: 2.17 The Harness framework is deprecated and will be moved out of the base diff --git a/docs/reference/api/ops-testing.rst b/docs/reference/api/ops-testing.rst index 76e4dba31..16da0e5d5 100644 --- a/docs/reference/api/ops-testing.rst +++ b/docs/reference/api/ops-testing.rst @@ -1,7 +1,7 @@ .. _ops_testing: `ops.testing` (was: Scenario) -============================ +============================= Install ops with the ``testing`` extra to use this API; for example: ``pip install ops[testing]`` From 968e636233cd686d4ff13562a30ea92e815a98b6 Mon Sep 17 00:00:00 2001 From: Tony Meyer Date: Wed, 4 Dec 2024 11:00:05 +1300 Subject: [PATCH 08/16] Move code to make linter happier. --- docs/custom_conf.py | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/docs/custom_conf.py b/docs/custom_conf.py index 579f7e30a..88f0e2e5d 100644 --- a/docs/custom_conf.py +++ b/docs/custom_conf.py @@ -24,13 +24,6 @@ from sphinx import addnodes from sphinx.util.docutils import SphinxDirective - -# FIXME: temporarily disable missing xrefs in this branch, until the missing -# links to Juju and Charmcraft are added. -suppress_warnings = [ - 'myst.xref_missing', -] - sys.path.insert(0, str(pathlib.Path(__file__).parent.parent)) # Pull in fix from https://github.com/sphinx-doc/sphinx/pull/11222/files to fix @@ -63,6 +56,12 @@ # features, it is sufficient to update the settings in the "Project # information" section. +# FIXME: temporarily disable missing xrefs in this branch, until the missing +# links to Juju and Charmcraft are added. +suppress_warnings = [ + 'myst.xref_missing', +] + ############################################################ ### Project information ############################################################ From 00b0f7a3f461b72923b4b2e102e4d74f64638af2 Mon Sep 17 00:00:00 2001 From: Tony Meyer Date: Wed, 4 Dec 2024 11:05:06 +1300 Subject: [PATCH 09/16] Update doc location. --- test/test_infra.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/test/test_infra.py b/test/test_infra.py index 47eaec39a..dc4f20761 100644 --- a/test/test_infra.py +++ b/test/test_infra.py @@ -76,7 +76,10 @@ def test_ops_testing_doc(): expected_names.add('Container') found_names: typing.Set[str] = set() - for test_doc in ('docs/harness.rst', 'docs/state-transition-testing.rst'): + for test_doc in ( + 'docs/reference/api/ops-testing-harness.rst', + 'docs/reference/api/ops-testing.rst', + ): with open(test_doc) as testing_doc: found_names.update({ line.split(prefix, 1)[1].strip() From 6cb125febed1b34e4d9732cdb313d793d8ba07fb Mon Sep 17 00:00:00 2001 From: Tony Meyer Date: Wed, 4 Dec 2024 11:11:12 +1300 Subject: [PATCH 10/16] Restore blank line. --- docs/custom_conf.py | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/custom_conf.py b/docs/custom_conf.py index 88f0e2e5d..19ae79fd5 100644 --- a/docs/custom_conf.py +++ b/docs/custom_conf.py @@ -24,6 +24,7 @@ from sphinx import addnodes from sphinx.util.docutils import SphinxDirective + sys.path.insert(0, str(pathlib.Path(__file__).parent.parent)) # Pull in fix from https://github.com/sphinx-doc/sphinx/pull/11222/files to fix From f2538b9b0a212ffbda60dfb59ef8b70ba1bf4f35 Mon Sep 17 00:00:00 2001 From: Tony Meyer Date: Wed, 4 Dec 2024 13:21:32 +1300 Subject: [PATCH 11/16] Add missing parenthesis. --- docs/index.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/index.md b/docs/index.md index 3ad8b6fac..03776bf33 100644 --- a/docs/index.md +++ b/docs/index.md @@ -20,7 +20,7 @@ The library ([available on PyPI](https://pypi.org/project/ops/)) provides: - {ref}`ops`, the API to respond to Juju events and manage the application; - {ref}`ops_pebble`, the Pebble client, a low-level API for Kubernetes containers; - {ref}`ops_testing`, the recommended API for unit testing charms; and -- {ref}`ops_testing_harness`, the legacy (deprecated API for unit testing charms. +- {ref}`ops_testing_harness`, the legacy (deprecated) API for unit testing charms. You can structure your charm however you like, but with the `ops` library, you get a framework that promotes consistency and readability by following best practices. It also helps you organise your code better by separating different aspects of the charm, such as managing the application's state, handling integrations with other services, and making the charm easier to test. From 9454b1c73875267d3a01f5ac9e7fe86a107526e7 Mon Sep 17 00:00:00 2001 From: Teodora Mihoc Date: Tue, 17 Dec 2024 09:40:22 +0100 Subject: [PATCH 12/16] docs: make requested changes --- docs/custom_conf.py | 6 +- .../charm-relation-interfaces.md | 0 docs/explanation/index.md | 3 + .../interface-tests.md | 0 docs/{reference => explanation}/testing.md | 22 +++ docs/howto/get-started-with-charm-testing.md | 6 +- docs/howto/index.md | 3 +- ...ument-your-charm-with-tracing-telemetry.md | 146 ------------------ docs/howto/manage-actions.md | 3 +- docs/howto/manage-configurations.md | 2 + docs/howto/manage-interfaces.md | 4 +- docs/howto/manage-leadership-changes.md | 3 +- docs/howto/manage-libraries.md | 7 +- docs/howto/manage-logs.md | 2 + docs/howto/manage-relations.md | 2 + docs/howto/manage-resources.md | 2 + docs/howto/manage-secrets.md | 2 + docs/howto/manage-storage.md | 2 + .../run-workloads-with-a-charm-kubernetes.md | 2 +- ...n-a-hooks-based-charm-into-an-ops-charm.md | 4 +- .../write-integration-tests-for-a-charm.md | 61 +++++--- docs/index.md | 21 +-- docs/reference/api/index.md | 12 -- docs/reference/index.md | 13 +- .../{api => }/ops-main-entrypoint.rst | 0 .../{api => }/ops-testing-harness.rst | 0 docs/reference/{api => }/ops-testing.rst | 0 docs/reference/{api => }/ops.rst | 0 docs/reference/{api => }/pebble.rst | 0 docs/reference/pytest-operator.md | 19 --- ... => create_a_minimal_kubernetes_charm.png} | Bin ...n-of-the-application-behind-your-charm.md# | 124 +++++++++++++++ .../create-a-minimal-kubernetes-charm.md | 26 ++-- .../index.md | 15 +- .../observe-your-charm-with-cos-lite.md | 3 +- .../preserve-your-charms-data.md | 13 +- .../publish-your-charm-on-charmhub.md | 4 +- .../write-your-first-machine-charm.md | 31 +++- 38 files changed, 292 insertions(+), 271 deletions(-) rename docs/{reference => explanation}/charm-relation-interfaces.md (100%) rename docs/{reference => explanation}/interface-tests.md (100%) rename docs/{reference => explanation}/testing.md (77%) delete mode 100644 docs/howto/instrument-your-charm-with-tracing-telemetry.md delete mode 100644 docs/reference/api/index.md rename docs/reference/{api => }/ops-main-entrypoint.rst (100%) rename docs/reference/{api => }/ops-testing-harness.rst (100%) rename docs/reference/{api => }/ops-testing.rst (100%) rename docs/reference/{api => }/ops.rst (100%) rename docs/reference/{api => }/pebble.rst (100%) delete mode 100644 docs/reference/pytest-operator.md rename docs/resources/{create_a_minimal_kubernetes_charm.jpg => create_a_minimal_kubernetes_charm.png} (100%) create mode 100644 docs/tutorial/from-zero-to-hero-write-your-first-kubernetes-charm/#expose-the-version-of-the-application-behind-your-charm.md# diff --git a/docs/custom_conf.py b/docs/custom_conf.py index 19ae79fd5..109c2ed8f 100644 --- a/docs/custom_conf.py +++ b/docs/custom_conf.py @@ -59,9 +59,9 @@ # FIXME: temporarily disable missing xrefs in this branch, until the missing # links to Juju and Charmcraft are added. -suppress_warnings = [ - 'myst.xref_missing', -] +#suppress_warnings = [ +# 'myst.xref_missing', +#] ############################################################ ### Project information diff --git a/docs/reference/charm-relation-interfaces.md b/docs/explanation/charm-relation-interfaces.md similarity index 100% rename from docs/reference/charm-relation-interfaces.md rename to docs/explanation/charm-relation-interfaces.md diff --git a/docs/explanation/index.md b/docs/explanation/index.md index 0238e87d8..23826e84a 100644 --- a/docs/explanation/index.md +++ b/docs/explanation/index.md @@ -4,6 +4,9 @@ ```{toctree} :maxdepth: 1 +charm-relation-interfaces +testing +interface-tests holistic-vs-delta-charms how-and-when-to-defer-events storedstate-uses-limitations diff --git a/docs/reference/interface-tests.md b/docs/explanation/interface-tests.md similarity index 100% rename from docs/reference/interface-tests.md rename to docs/explanation/interface-tests.md diff --git a/docs/reference/testing.md b/docs/explanation/testing.md similarity index 77% rename from docs/reference/testing.md rename to docs/explanation/testing.md index 0ba3df7b1..0e2dbb8b6 100644 --- a/docs/reference/testing.md +++ b/docs/explanation/testing.md @@ -83,6 +83,28 @@ When writing an integration test, it is not sufficient to simply check that Juju - [pytest-operator](https://github.com/charmed-kubernetes/pytest-operator) and/or [`zaza`](https://github.com/openstack-charmers/zaza) +(pytest-operator)= +### `pytest-operator` + +`pytest-operator` is a Python library that provides Juju plugins for the generic Python library `pytest` to facilitate the {ref}`integration testing ` of charms. + +> See more: [`pytest-operator`](https://github.com/charmed-kubernetes/pytest-operator) + +It builds a fixture called `ops_test` that helps you interact with Juju through constructs that wrap around [`python-libjuju` ](https://pypi.org/project/juju/). + +> See more: +> - [`pytest-operator` > `ops_test`](https://github.com/charmed-kubernetes/pytest-operator/blob/main/docs/reference.md#ops_test) +> - [`pytest` > Fixtures](https://docs.pytest.org/en/6.2.x/fixture.html) + +It also provides convenient markers and command line parameters (e.g., the `@pytest.mark.skip_if_deployed` marker in combination with the `--no-deploy` configuration helps you skip, e.g., a deployment test in the case where you already have a deployment). + + +> See more: +> - [`pytest-operator` > Markers](https://github.com/charmed-kubernetes/pytest-operator/blob/main/docs/reference.md#markers) +> - [`pytest-operator` > Command line parameters](https://github.com/charmed-kubernetes/pytest-operator/blob/main/docs/reference.md#command-line-parameters) + + + **Examples.** - [https://github.com/canonical/prometheus-k8s-operator/blob/main/tests/integration/test_charm.py](https://github.com/canonical/prometheus-k8s-operator/blob/main/tests/integration/test_charm.py) diff --git a/docs/howto/get-started-with-charm-testing.md b/docs/howto/get-started-with-charm-testing.md index 0c2edf63f..7172d5749 100644 --- a/docs/howto/get-started-with-charm-testing.md +++ b/docs/howto/get-started-with-charm-testing.md @@ -6,7 +6,7 @@ Testing charm code is an essential part of charming. Here we will see how to get **What you'll need:** - knowledge of testing in general - knowledge of Juju and charms -- knowledge of the Juju models and events, esp. the data involved in a charm's lifecycle (e.g. see [Talking to a workload control flow from A to Z]()) +- knowledge of the Juju models and events, esp. the data involved in a charm's lifecycle **What you will learn:** - What are the starting points for adding tests to a charm? @@ -77,7 +77,9 @@ If the charm is a machine charm, workload operation calls can be done directly, "Juju operations" are the most 'meta' of them all: they do not affect the workload in and of itself, but they share data which is meant to affect the operation of *other* charms that this charm is integrated with. + ### What we are testing when we unit-test @@ -120,7 +122,9 @@ There are two ways to initialize a harnessed charm: * When a charm is deployed, it goes through the Setup phase, a fixed sequence of events. `Harness` has a method, `begin_with_initial_hooks()`, that runs this sequence. * Alternatively, you can initialise the charm by calling `begin()`. This will instantiate the charm without firing any Setup phase event. + After the Setup phase, the charm goes into Operation. To test operation-phase-related events, the harness provides some methods to simulate the most common scenarios. For example: diff --git a/docs/howto/index.md b/docs/howto/index.md index 8fec3f173..8269419a2 100644 --- a/docs/howto/index.md +++ b/docs/howto/index.md @@ -1,4 +1,4 @@ -(howto-guides)= +(how-to-guides)= # How-to guides ```{toctree} @@ -22,7 +22,6 @@ Get started with charm testing Write unit tests for a charm Write scenario tests for a charm Write integration tests for a charm -Instrument your charm with tracing telemetry Turn a hooks-based charm into an ops charm ``` diff --git a/docs/howto/instrument-your-charm-with-tracing-telemetry.md b/docs/howto/instrument-your-charm-with-tracing-telemetry.md deleted file mode 100644 index 04b794c51..000000000 --- a/docs/howto/instrument-your-charm-with-tracing-telemetry.md +++ /dev/null @@ -1,146 +0,0 @@ -(how-to-instrument-your-charm-with-tracing-telemetry)= -# How to instrument your charm with tracing telemetry - -```{note} - -This howto has been incorporated into [the Tempo charm docs](https://charmhub.io/topics/charmed-tempo-ha/instrument%20your%20charm). - -``` - - -In order to instrument a charm with tracing telemetry, you will need to: - -1. Set up a model with `cos-lite`: - - -```bash -juju add-model clite -juju deploy cos-lite --trust -``` - -> See more: [Charmhub | `cos-lite`](https://charmhub.io/cos-lite) - -2. Deploy `tempo`: - -```bash -juju deploy tempo-k8s -``` - -> See more: [Charmhub | `tempo-k8s`](https://charmhub.io/tempo-k8s) - -3. Integrate tempo with `cos-lite`: - -```bash -jhack imatrix fill -``` - -> See more: {ref}``jhack` `, [GitHub | `jhack` > `imatrix` > `fill`](https://github.com/PietroPasotti/jhack#fill) - -Alternatively, you can integrate manually by: -``` -juju integrate tempo:logging loki:logging -juju integrate tempo:ingress traefik:ingress -juju integrate grafana:grafana-source tempo:grafana-source -juju integrate prometheus:metrics-endpoint, tempo:metrics-endpoint -juju integrate grafana:grafana-dashboard, tempo:grafana-dashboard -juju integrate traefik:tracing, tempo:tracing -juju integrate prometheus:tracing, tempo:tracing -``` - -```{note} - At some point there will be an overlay bundle to deploy `cos-lite` + `tempo` and integrate them; see [this PR](https://github.com/canonical/cos-lite-bundle/pull/79) to follow the progress on the overlay. -``` - -At this point you should have a working `cos-lite` plus `tempo` deployment; something like this: -![image|690x333](upload://7ohHj2JBT9Zaezsgx6p3a56n5cO.png) - -4. Fetch the `charm_tracing` and the `tracing` libs: -```bash -charmcraft fetch-lib charms.tempo-k8s.v1.charm_tracing -charmcraft fetch-lib charms.tempo-k8s.v2.tracing -``` - -> See more: {ref}``charmcraft fetch-lib` `, [Charmhub | `tempo-k8s` > Libraries > `charm-tracing`](https://charmhub.io/tempo-k8s/libraries/charm_tracing), [Charmhub | `tempo-k8s` > Libraries > `tracing`](https://charmhub.io/tempo-k8s/libraries/tracing) - -5. Add opentelemetry exporter dependency used by `charm_tracing` to your `requirements.txt`: - -``` -opentelemetry-exporter-otlp-proto-http>=1.21.0 -``` - -6. Add an integration with `tempo`: - -```yaml -# in charmcraft.yaml -provides: - tracing: - interface: tracing - limit: 1 -``` - -> See more: [File `charmcraft.yaml` > `provides`]() - -7. Instrument your charm code: - -```python -# in /your/charm/project/src/charm.py -from charms.tempo_k8s.v1.charm_tracing import trace_charm -from charms.tempo_k8s.v2.tracing import TracingEndpointRequirer - -@trace_charm(tracing_endpoint="tracing_endpoint") -class MyCharm(CharmBase): - def __init__(self, *args): - super().__init__(*args) - # Add a provider wrapper for the tracing endpoint. - self.tracing = TracingEndpointRequirer(self, protocols=["otlp_http"]) - - @property - def tracing_endpoint(self) -> Optional[str]: - """Tempo endpoint for charm tracing.""" - if self.tracing.is_ready(): - return self.tracing.get_endpoint("otlp_http") - return None -``` -```{note} - -By default, the traces generated by this charm will be tagged with service name equal to the class name of the charm, so `"MyCharm"` in this case. You override this default by passing -`service_name="foo"` to `trace_charm`. - -``` - - -8. Pack, deploy, and integrate your charm with `cos-lite`: - -```bash -charmcraft pack -juju deploy ./my-charm-operator_ubuntu-20.04-amd64.charm mycharm \ - $(yq eval '.resources | to_entries | map(select(.value.upstream-source != null) | "--resource " + .key + "=" + .value.upstream-source) | join(" ")' charmcraft.yaml) -juju integrate mycharm:tracing tempo-k8s:tracing -``` - -```{note} - -`cos-lite` is a kubernetes bundle. `tempo` is a Kubernetes charm. - -``` - -> See more: [`charmcraft pack` `, [Juju | `juju deploy`](https://juju.is/docs/juju/juju-deploy), [Juju | `juju integrate`](https://juju.is/docs/juju/juju-integrate) - -9. View the traces for the charm. - -Open the Grafana dashboard in a browser ([see here](https://github.com/canonical/grafana-k8s-operator) for more detailed instructions). - -Next, navigate to the traces for your charm: -- go to `Explore` and select the Tempo datasource. -- pick the `service_name` you gave to MyCharm above (the default is the application name) to see the traces for that charm -![image|690x505](upload://rAZOurgeYtFfymdB1V0qzKg7HcW.png) -- click on a trace ID to visualize it. For example, this is the trace for an `update-status` event on the Tempo charm itself: -![image|690x492](upload://6lufy3oZUYDkwxrnufQJKf5tpGN.png) - - -## Mapping events to traces with jhack tail -`jhack tail` supports a `-t` option to show the trace IDs associated with a charm execution: - -![image|607x326](upload://1UUxRnTkBR3hkCPfWd0sd4AXqfJ.png) - -This means that you can tail a charm, grab the trace id from tail, put it in the grafana dashboard query and get to the trace in no time. diff --git a/docs/howto/manage-actions.md b/docs/howto/manage-actions.md index 46b00d726..c9e8ed5f2 100644 --- a/docs/howto/manage-actions.md +++ b/docs/howto/manage-actions.md @@ -1,8 +1,9 @@ (manage-actions)= # How to manage actions + ## Implement the feature diff --git a/docs/howto/manage-configurations.md b/docs/howto/manage-configurations.md index aa788d44e..d80854f85 100644 --- a/docs/howto/manage-configurations.md +++ b/docs/howto/manage-configurations.md @@ -1,7 +1,9 @@ (manage-configurations)= # Manage configurations + ## Implement the feature diff --git a/docs/howto/manage-interfaces.md b/docs/howto/manage-interfaces.md index 588ddadae..cec28e89d 100644 --- a/docs/howto/manage-interfaces.md +++ b/docs/howto/manage-interfaces.md @@ -332,7 +332,7 @@ You should see: In particular, pay attention to the `provider` field. If it says `` then there is something wrong with your setup, and the collector isn't able to find your test or identify it as a valid test. -Similarly, you can add tests for requirer in `./interfaces/my_fancy_database/v0/interface_tests/test_requirer.py`. Don't forget to edit the `interface.yaml` file in the "requirers" section to add the name of the charm and the URL. See the "Edit `interface.yaml`" section in the previous how-to guide [How to register an interface]() for more detail on editing `interface.yaml`. [Here](https://github.com/IronCore864/charm-relation-interfaces/tree/my-fancy-database/interfaces/my_fancy_database/v0) is an example of tests for requirers added. +Similarly, you can add tests for requirer in `./interfaces/my_fancy_database/v0/interface_tests/test_requirer.py`. Don't forget to edit the `interface.yaml` file in the "requirers" section to add the name of the charm and the URL. See the "Edit `interface.yaml`" section in the previous how-to guide "How to register an interface" for more detail on editing `interface.yaml`. [Here](https://github.com/IronCore864/charm-relation-interfaces/tree/my-fancy-database/interfaces/my_fancy_database/v0) is an example of tests for requirers added. ### Merge in charm-relation-interfaces @@ -451,7 +451,7 @@ INFO:root:Running tests for role: provider } ``` -For reference, [here](https://github.com/IronCore864/my-fancy-database-operator) is an example of a bare minimum `my-fancy-database-operator` charm to make the test pass. In the charm, application relation data and unit relation data are set according to our definition (see the beginning part of the previous how-to guide [How to register an interface](). +For reference, [here](https://github.com/IronCore864/my-fancy-database-operator) is an example of a bare minimum `my-fancy-database-operator` charm to make the test pass. In the charm, application relation data and unit relation data are set according to our definition (see the beginning part of the previous how-to guide "How to register an interface". ### Troubleshooting and debugging the tests diff --git a/docs/howto/manage-leadership-changes.md b/docs/howto/manage-leadership-changes.md index 634487675..1546f056b 100644 --- a/docs/howto/manage-leadership-changes.md +++ b/docs/howto/manage-leadership-changes.md @@ -1,8 +1,9 @@ (manage-leadership-changes)= # Manage leadership changes + ## Implement response to leadership changes diff --git a/docs/howto/manage-libraries.md b/docs/howto/manage-libraries.md index b7ad88929..11d04c0f2 100644 --- a/docs/howto/manage-libraries.md +++ b/docs/howto/manage-libraries.md @@ -1,8 +1,9 @@ (manage-libraries)= # Manage libraries -> See first: [`juju` | Library](), [`charmcraft` | Manage libraries]() - + ## Write a library When you're writing libraries, instead of callbacks, you can use custom events; that'll result in a more `ops`-native-feeling API. A custom event is, from a technical standpoint, an EventBase subclass that can be emitted at any point throughout the charm's lifecycle. These events are therefore totally unknown to Juju. They are essentially charm-internal, and can be useful to abstract certain conditional workflows and wrap the toplevel Juju event so it can be observed independently. @@ -209,7 +210,9 @@ def test_my_object_data(context, endpoint, n_relations): Fetch the library. + In your `src/charm.py`, observe the custom events it puts at your disposal. For example, a database library may have provided a `database_relation_ready` event -- a high-level wrapper around the relevant `juju` relation events -- so you use it to manage the database integration in your charm as below: diff --git a/docs/howto/manage-logs.md b/docs/howto/manage-logs.md index 2073b705e..c2e9fe6e0 100644 --- a/docs/howto/manage-logs.md +++ b/docs/howto/manage-logs.md @@ -1,7 +1,9 @@ (how-to-log-a-message-in-a-charm)= # How to log a message in a charm + ## Implement the feature diff --git a/docs/howto/manage-resources.md b/docs/howto/manage-resources.md index d23143f40..d32cef5fa 100644 --- a/docs/howto/manage-resources.md +++ b/docs/howto/manage-resources.md @@ -1,7 +1,9 @@ (manage-resources)= # How to manage resources + ## Implement the feature diff --git a/docs/howto/manage-secrets.md b/docs/howto/manage-secrets.md index 3bfea3532..4c304f3a5 100644 --- a/docs/howto/manage-secrets.md +++ b/docs/howto/manage-secrets.md @@ -1,7 +1,9 @@ (manage-secrets)= # How to manage secrets + > Added in `ops 2.0.0`, `juju 3.0.2` diff --git a/docs/howto/manage-storage.md b/docs/howto/manage-storage.md index 1d39ac19e..6cb867ebc 100644 --- a/docs/howto/manage-storage.md +++ b/docs/howto/manage-storage.md @@ -1,7 +1,9 @@ (manage-storage)= # How to manage storage + ## Implement the feature diff --git a/docs/howto/run-workloads-with-a-charm-kubernetes.md b/docs/howto/run-workloads-with-a-charm-kubernetes.md index e89833c41..b06e80feb 100644 --- a/docs/howto/run-workloads-with-a-charm-kubernetes.md +++ b/docs/howto/run-workloads-with-a-charm-kubernetes.md @@ -5,7 +5,7 @@ The recommended way to create charms for Kubernetes is using the sidecar pattern Pebble is a lightweight, API-driven process supervisor designed for use with charms. If you specify the `containers` field in a charm's `charmcraft.yaml`, Juju will deploy the charm code in a sidecar container, with Pebble running as the workload container's `ENTRYPOINT`. -When the workload container starts up, Juju fires a [`PebbleReadyEvent`](https://ops.readthedocs.io/en/latest/#ops.PebbleReadyEvent), which can be handled using [`Framework.observe`](https://ops.readthedocs.io/en/latest/#ops.Framework.observe) as shown in [Framework Constructs under "Containers"](). This gives the charm author access to `event.workload`, a [`Container`](https://ops.readthedocs.io/en/latest/#ops.Container) instance. +When the workload container starts up, Juju fires a [`PebbleReadyEvent`](https://ops.readthedocs.io/en/latest/#ops.PebbleReadyEvent), which can be handled using [`Framework.observe`](https://ops.readthedocs.io/en/latest/#ops.Framework.observe). This gives the charm author access to `event.workload`, a [`Container`](https://ops.readthedocs.io/en/latest/#ops.Container) instance. The `Container` class has methods to modify the Pebble configuration "plan", start and stop services, read and write files, and run commands. These methods use the Pebble API, which communicates from the charm container to the workload container using HTTP over a Unix domain socket. diff --git a/docs/howto/turn-a-hooks-based-charm-into-an-ops-charm.md b/docs/howto/turn-a-hooks-based-charm-into-an-ops-charm.md index 82ea42969..aabacff51 100644 --- a/docs/howto/turn-a-hooks-based-charm-into-an-ops-charm.md +++ b/docs/howto/turn-a-hooks-based-charm-into-an-ops-charm.md @@ -1,7 +1,9 @@ (turn-a-hooks-based-charm-into-an-ops-charm)= # How to turn a hooks-based charm into an ops charm + Suppose you have a hooks-based charm and you decide to rewrite it using the Ops framework in Python. @@ -24,7 +26,7 @@ We start by looking at the charm we intend to translate; as an example, we will From the charm root directory we see: -```bash +```text $ tree . . ├── charmcraft.yaml diff --git a/docs/howto/write-integration-tests-for-a-charm.md b/docs/howto/write-integration-tests-for-a-charm.md index 1eff1607a..a0801d2cf 100644 --- a/docs/howto/write-integration-tests-for-a-charm.md +++ b/docs/howto/write-integration-tests-for-a-charm.md @@ -19,7 +19,9 @@ The instructions all use the Juju `python-libjuju` client, either through the `p In order to run integrations tests you will need to have your environment set up with `tox` installed. + ## Prepare the `tox.ini` configuration file @@ -42,7 +44,7 @@ commands = {[vars]tests_path}/integration ``` -

Create a test file

 +## Create a test file By convention, integration tests are kept in the charm’s source tree, in a directory called `tests/integration`. @@ -55,7 +57,7 @@ from pytest_operator.plugin import OpsTest The `ops_test` fixture is your entry point to the `pytest-operator` library, and the preferred way of interacting with Juju in integration tests. This fixture will create a model for each test file -- if you write two tests that should not share a model, make sure to place them in different files. -

Build your tests

 +## Build your tests ```{note} @@ -63,7 +65,7 @@ Use `pytest` custom markers to toggle which types of tests are being run so you ``` -

Test build and deploy

 +### Test build and deploy To build and deploy the current charm, in your integration test file, add the function below: @@ -79,7 +81,7 @@ async def test_build_and_deploy(ops_test: OpsTest): Tests run sequentially in the order they are written in the file. It can be useful to put tests that build and deploy applications in the top of the file as the applications can be used by other tests. For that reason, adding extra checks or `asserts` in this test is not recommended. -The decorator `@pytest.mark.abort_on_fail` abort all next tests if something goes wrong. With the decorator `@pytest.mark.skip_if_deployed` you can skip that test if a `--model` is passed as a command line parameter (see [Run your tests](#run-tests) for more information). +The decorator `@pytest.mark.abort_on_fail` abort all next tests if something goes wrong. With the decorator `@pytest.mark.skip_if_deployed` you can skip that test if a `--model` is passed as a command line parameter (see {ref}`run-your-tests` for more information). `ops_test.build_charm` builds the charm with charmcraft. `ops_test.model` is an instance of `python-libjuju` 's [Model](https://pythonlibjuju.readthedocs.io/en/latest/api/juju.model.html#juju.model.Model) class that reference the active model tracked by `pytest-operator` for the current module. @@ -96,9 +98,11 @@ As an alternative to `wait_for_idle`, you can explicitly block until the applica > - [`pytest-operator` | `ops_test.build_charm`](https://github.com/charmed-kubernetes/pytest-operator/blob/ab50fc20320d3ea3d8a37495f92a004531a4023f/pytest_operator/plugin.py#L1020) > - [`python-libjuju` | `model.deploy `](https://github.com/juju/python-libjuju/blob/2581b0ced1df6201c6b7fd8cc0b20dcfa9d97c51/juju/model.py#L1658) -

Deploy your charm with resources

 +### Deploy your charm with resources + A charm can require `file` or `oci-image` `resources` to work, that can be provided to `ops_test.model.deploy`. In Charmhub, resources have revision numbers. For file resources already stored in Charmhub, you can use `ops_test.download_resources`: @@ -130,7 +134,7 @@ For `oci-images` you can reference an image registry. > - [`python-libjuju` | `model.deploy`](https://github.com/juju/python-libjuju/blob/2581b0ced1df6201c6b7fd8cc0b20dcfa9d97c51/juju/model.py#L1658) -

Test a relation

 +### Test a relation To test an integration between two applications, you can just integrate them through the model. Both applications have to be deployed beforehand. @@ -150,9 +154,12 @@ async def test_my_integration(ops_test: OpsTest): > See more: [`python-libjuju` | `model.integrate`](https://github.com/juju/python-libjuju/blob/2581b0ced1df6201c6b7fd8cc0b20dcfa9d97c51/juju/model.py#L1476) -

Test a configuration

 +### Test a configuration + + You can set a configuration option in your application and check its results. @@ -171,13 +178,15 @@ async def test_config_changed(ops_test: OpsTest): -

Test an action

 +### Test an action + You can execute an action on a unit and get its results. -``` +```text async def test_run_action(ops_test: OpsTest): action_register_user = await ops_test.model.applications["myapp"].units[0].run_action("register-user", username="ubuntu") await action_register_user.wait() @@ -188,13 +197,13 @@ async def test_run_action(ops_test: OpsTest): > See also: [python-libjuju | unit.run_action](https://github.com/juju/python-libjuju/blob/2581b0ced1df6201c6b7fd8cc0b20dcfa9d97c51/juju/unit.py#L274) -

Interact with the workload

 +### Interact with the workload To interact with the workload, you need to have access to it. This is dependent on many aspects of your application, environment and network topology. You can get information from your application or unit addresses using `await ops_test.model.get_status`. That way, if your application exposes a public address you can reference it. You can also try to connect to a unit address or public address. -``` +```text async def test_workload_connectivity(ops_test: OpsTest): status = await ops_test.model.get_status() address = status.applications['my_app'].public_address @@ -210,15 +219,17 @@ How you can connect to a private or public address is dependent on your configur > Example implementations: [mongodb-k8s-operator](https://github.com/canonical/mongodb-k8s-operator/blob/8b9ebbee3f225ca98175c25781f1936dc4a62a7d/tests/integration/metrics_tests/test_metrics.py#L33), [tempo-k8s-operator](https://github.com/canonical/tempo-k8s-operator/blob/78a1143d99af99a1a56fe9ff82b1a3563e4fd2f7/tests/integration/test_integration.py#L69), [synapse](https://github.com/canonical/synapse-operator/blob/eb44f4959a00040f08b98470f8b17cae4cc616da/tests/integration/conftest.py#L170) + -

Run a subprocess command within Juju context

 +### Run a subprocess command within Juju context You can run a command within the Juju context with: -``` +```text ... command = ["microk8s", "version"] returncode, stdout, stderr = await ops_test.run(*command, check=True) @@ -242,7 +253,7 @@ so you don't need to include the `-m` parameter. > - [`pytest-operator` | `run`](https://github.com/charmed-kubernetes/pytest-operator/blob/ab50fc20320d3ea3d8a37495f92a004531a4023f/pytest_operator/plugin.py#L576) > - [`pytest-operator` | `juju`](https://github.com/charmed-kubernetes/pytest-operator/blob/ab50fc20320d3ea3d8a37495f92a004531a4023f/pytest_operator/plugin.py#L624) -

Use several models

 +### Use several models You can use `pytest-operator` with several models, in the same cloud or in different clouds. This way you can, for example, integrate machine charms @@ -274,25 +285,26 @@ Using the new alias, you can switch context to the new created model, similar to > Example implementations: [`charm-kubernetes-autoscaler`](https://github.com/charmed-kubernetes/charm-kubernetes-autoscaler/blob/8f4ddf5d66802ade73ed3aab2bb8d09fd9e4d63a/tests/integration/test_kubernetes_autoscaler.py#L31) + - -

Deploy a bundle

 +### Deploy a bundle ```{note} -It is not recommended to use `ops_test.build_bundle` and `ops_test.deploy_bundle` until this [issue](https://github.com/charmed-kubernetes/pytest-operator/issues/98) is closed, as it uses `juju-bundle` which is outdated. You can deploy bundles using `ops_test.model.deploy` or [`ops_test.juju`](). +It is not recommended to use `ops_test.build_bundle` and `ops_test.deploy_bundle` until this [issue](https://github.com/charmed-kubernetes/pytest-operator/issues/98) is closed, as it uses `juju-bundle` which is outdated. You can deploy bundles using `ops_test.model.deploy` or `ops_test.juju`. ``` -

Render bundles and charms

 +### Render bundles and charms `pytest-operator` has utilities to template your charms and bundles using Jinja2. @@ -324,7 +336,7 @@ async def test_build_and_deploy_bundle(ops_test: OpsTest): -

Speed up `update_status` with `fast_forward`

 +### Speed up `update_status` with `fast_forward` If your charm code depends on the `update_status` event, you can speed up its firing rate with `fast_forward`. Inside the new async context you can put any code that will benefit from the new refresh rate so your test may execute faster. @@ -344,8 +356,8 @@ firing rate with `fast_forward`. Inside the new async context you can put any co > See more: > - [`pytest-operator` | `fast_forward`](https://github.com/charmed-kubernetes/pytest-operator/blob/ab50fc20320d3ea3d8a37495f92a004531a4023f/pytest_operator/plugin.py#L1400) - -

Run your tests

 +(run-your-tests)= +## Run your tests By default you can run all your tests with: @@ -370,13 +382,15 @@ There are different ways of specifying a subset of tests to run using `pytest`. tox -e integration -- tests/integration/test_charm.py -k "not test_one" ``` + > See more: > - [`pytest-operator` | `skip_if_deployed`](https://github.com/charmed-kubernetes/pytest-operator/blob/ab50fc20320d3ea3d8a37495f92a004531a4023f/pytest_operator/plugin.py#L139) > - [`pytest | How to invoke pytest`](https://docs.pytest.org/en/7.1.x/how-to/usage.html) -

Generate crash dumps

 +## Generate crash dumps To generate crash dumps, you need the `juju-crashdump` tool . @@ -394,6 +408,3 @@ tox -e integration -- --crash-dump=always --crash-dump-output=/tmp > See more: > - [`juju-crashdump`](https://github.com/juju/juju-crashdump) > - [`pytest-operator` | `--crash-dump`](https://github.com/charmed-kubernetes/pytest-operator/blob/ab50fc20320d3ea3d8a37495f92a004531a4023f/pytest_operator/plugin.py#L97) - - -> Contributors: @natalia-nowakowska , @javierdelapuente diff --git a/docs/index.md b/docs/index.md index 03776bf33..36acd1cb7 100644 --- a/docs/index.md +++ b/docs/index.md @@ -1,4 +1,4 @@ -# Ops (`ops`) +# Ops documentation ```{toctree} :maxdepth: 2 @@ -12,19 +12,20 @@ explanation/index -The Ops (`ops`) library is a Python framework for writing and testing Juju charms. +The Ops library (`ops`) is a Python framework for writing and testing Juju charms. -The library ([available on PyPI](https://pypi.org/project/ops/)) provides: +> [See it on PyPI](https://pypi.org/project/ops/) -- {ref}`ops_main_entry_point`, used to initialise and run your charm; -- {ref}`ops`, the API to respond to Juju events and manage the application; -- {ref}`ops_pebble`, the Pebble client, a low-level API for Kubernetes containers; -- {ref}`ops_testing`, the recommended API for unit testing charms; and -- {ref}`ops_testing_harness`, the legacy (deprecated) API for unit testing charms. +The library provides: + +- {ref}`ops_main_entry_point`, used to initialise and run your charm +- {ref}`ops`, the API to respond to Juju events and manage the application +- {ref}`ops_pebble`, the Pebble client, a low-level API for Kubernetes containers +- {ref}`ops_testing`, the recommended API for unit testing charms +- {ref}`ops_testing_harness`, the deprecated API for unit testing charms You can structure your charm however you like, but with the `ops` library, you get a framework that promotes consistency and readability by following best practices. It also helps you organise your code better by separating different aspects of the charm, such as managing the application's state, handling integrations with other services, and making the charm easier to test. -If you're serious about charming, you *must* use `ops`! --------- @@ -83,7 +84,7 @@ As a community we adhere to the Ubuntu code of conduct. Discourse is the go-to forum for all questions Ops. * **[Join our online chat](https://matrix.to/#/#charmhub-ops:ubuntu.com)**: -Meet us in the `#ops` channel on Matrix. +Meet us in the #charmhub-charmdev channel on Matrix. * **[Report bugs](https://github.com/canonical/operator/)**: We want to know about the problems so we can fix them. diff --git a/docs/reference/api/index.md b/docs/reference/api/index.md deleted file mode 100644 index 2a5c77f38..000000000 --- a/docs/reference/api/index.md +++ /dev/null @@ -1,12 +0,0 @@ -(ops-api-index)= -# `ops` API reference - -```{toctree} -:maxdepth: 2 - -ops-main-entrypoint -ops -pebble -ops-testing -ops-testing-harness -``` diff --git a/docs/reference/index.md b/docs/reference/index.md index 8fbcc6269..196c5eb6c 100644 --- a/docs/reference/index.md +++ b/docs/reference/index.md @@ -2,13 +2,12 @@ # Reference ```{toctree} -:maxdepth: 2 - -charm-relation-interfaces -api/index -pytest-operator -testing -interface-tests +:maxdepth: 1 +ops-main-entrypoint +ops +pebble +ops-testing +ops-testing-harness ``` diff --git a/docs/reference/api/ops-main-entrypoint.rst b/docs/reference/ops-main-entrypoint.rst similarity index 100% rename from docs/reference/api/ops-main-entrypoint.rst rename to docs/reference/ops-main-entrypoint.rst diff --git a/docs/reference/api/ops-testing-harness.rst b/docs/reference/ops-testing-harness.rst similarity index 100% rename from docs/reference/api/ops-testing-harness.rst rename to docs/reference/ops-testing-harness.rst diff --git a/docs/reference/api/ops-testing.rst b/docs/reference/ops-testing.rst similarity index 100% rename from docs/reference/api/ops-testing.rst rename to docs/reference/ops-testing.rst diff --git a/docs/reference/api/ops.rst b/docs/reference/ops.rst similarity index 100% rename from docs/reference/api/ops.rst rename to docs/reference/ops.rst diff --git a/docs/reference/api/pebble.rst b/docs/reference/pebble.rst similarity index 100% rename from docs/reference/api/pebble.rst rename to docs/reference/pebble.rst diff --git a/docs/reference/pytest-operator.md b/docs/reference/pytest-operator.md deleted file mode 100644 index b86ec3767..000000000 --- a/docs/reference/pytest-operator.md +++ /dev/null @@ -1,19 +0,0 @@ -(pytest-operator)= -# `pytest-operator` - -`pytest-operator` is a Python library that provides Juju plugins for the generic Python library `pytest` to facilitate the {ref}`integration testing ` of charms. - -> See more: [`pytest-operator`](https://github.com/charmed-kubernetes/pytest-operator) - -It builds a fixture called `ops_test` that helps you interact with Juju through constructs that wrap around [`python-libjuju` ](https://pypi.org/project/juju/). - -> See more: -> - [`pytest-operator` > `ops_test`](https://github.com/charmed-kubernetes/pytest-operator/blob/main/docs/reference.md#ops_test) -> - [`pytest` > Fixtures](https://docs.pytest.org/en/6.2.x/fixture.html) - -It also provides convenient markers and command line parameters (e.g., the `@pytest.mark.skip_if_deployed` marker in combination with the `--no-deploy` configuration helps you skip, e.g., a deployment test in the case where you already have a deployment). - - -> See more: -> - [`pytest-operator` > Markers](https://github.com/charmed-kubernetes/pytest-operator/blob/main/docs/reference.md#markers) -> - [`pytest-operator` > Command line parameters](https://github.com/charmed-kubernetes/pytest-operator/blob/main/docs/reference.md#command-line-parameters) diff --git a/docs/resources/create_a_minimal_kubernetes_charm.jpg b/docs/resources/create_a_minimal_kubernetes_charm.png similarity index 100% rename from docs/resources/create_a_minimal_kubernetes_charm.jpg rename to docs/resources/create_a_minimal_kubernetes_charm.png diff --git a/docs/tutorial/from-zero-to-hero-write-your-first-kubernetes-charm/#expose-the-version-of-the-application-behind-your-charm.md# b/docs/tutorial/from-zero-to-hero-write-your-first-kubernetes-charm/#expose-the-version-of-the-application-behind-your-charm.md# new file mode 100644 index 000000000..2cc19f05e --- /dev/null +++ b/docs/tutorial/from-zero-to-hero-write-your-first-kubernetes-charm/#expose-the-version-of-the-application-behind-your-charm.md# @@ -0,0 +1,124 @@ +(expose-the-version-of-the-application-behind-your-charm)= +# Expose the version of the application behind your charm + +> {ref}`From Zero to Hero: Write your first Kubernetes charm ` > Expose the version of the application behind your charm +> +> **See previous: {ref}`Make your charm configurable `** + +````{important} + +This document is part of a series, and we recommend you follow it in sequence. However, you can also jump straight in by checking out the code from the previous branches: + +``` +git clone https://github.com/canonical/juju-sdk-tutorial-k8s.git +cd juju-sdk-tutorial-k8s +git checkout 02_make_your_charm_configurable +git checkout -b 03_set_workload_version +``` + +```` + +In this chapter of the tutorial you will learn how to expose the version of the application (workload) run by the charm -- something that a charm user might find it useful to know. + +**Contents:** + +1. [Define functions to collect the workload application version and set it in the charm](#heading--define-functions-to-collect-the-workload-application-version-and-set-it-in-the-charm) +1. [Declare Python dependencies](#heading--declare-python-dependencies) +1. [Validate your charm](#heading--validate-your-charm) +1. [Review the final code](#heading--review-the-final-code) + +

Define functions to collect the workload application version and set it in the charm

 + +As a first step we need to add two helper functions that will send an HTTP request to our application to get its version. If the container is available, we can send a request using the `requests` Python library and then add class methods to parse the JSON output to get a version string, as shown below: + +- Import the `requests` Python library: + +``` +import requests +``` + +- Add the following class methods: + + +```python +@property +def version(self) -> str: + """Reports the current workload (FastAPI app) version.""" + try: + if self.container.get_services(self.pebble_service_name): + return self._request_version() + # Catching Exception is not ideal, but we don't care much for the error here, and just + # default to setting a blank version since there isn't much the admin can do! + except Exception as e: + logger.warning("unable to get version from API: %s", str(e), exc_info=True) + return "" + +def _request_version(self) -> str: + """Helper for fetching the version from the running workload using the API.""" + resp = requests.get(f"http://localhost:{self.config['server-port']}/version", timeout=10) + return resp.json()["version"] +``` + +Next, we need to update the `_update_layer_and_restart` method to set our workload version. Insert the following lines before setting `ActiveStatus`: + +```python +# Add workload version in Juju status. +self.unit.set_workload_version(self.version) +``` + +

Declare Python dependencies

 + + +Since we've added a third party Python dependency into our project, we need to list it in `requirements.txt`. Edit the file to add the following line: + +``` +requests~=2.28 +``` + +Next time you run `charmcraft` it will fetch this new dependency into the charm package. + + +

Validate your charm

 + +We've exposed the workload version behind our charm. Let's test that it's working! + +First, repack and refresh your charm: + +``` +charmcraft pack +juju refresh \ + --path="./demo-api-charm_ubuntu-22.04-amd64.charm" \ + demo-api-charm --force-units --resource \ + demo-server-image=ghcr.io/canonical/api_demo_server:1.0.1 +``` + +Our charm should fetch the application version and forward it to `juju`. Run `juju status` to check: + +```text +juju status +``` + +Indeed, the version of our workload is now displayed -- see the App block, the Version column: + +```text +Model Controller Cloud/Region Version SLA Timestamp +charm-model tutorial-controller microk8s/localhost 3.0.0 unsupported 12:37:27+01:00 + +App Version Status Scale Charm Channel Rev Address Exposed Message +demo-api-charm 1.0.1 active 1 demo-api-charm 0 10.152.183.233 no + +Unit Workload Agent Address Ports Message +demo-api-charm/0* active idle 10.1.157.75 +``` + +

Review the final code

 + + +For the full code see: [03_set_workload_version](https://github.com/canonical/juju-sdk-tutorial-k8s/tree/03_set_workload_version) + +For a comparative view of the code before and after this doc see: [Comparison](https://github.com/canonical/juju-sdk-tutorial-k8s/compare/02_make_your_charm_configurable...03_set_workload_version) + + +> **See next: {ref}`Integrate your charm with PostgreSQL `** + +> Contributors: @beliaev-maksim \ No newline at end of file diff --git a/docs/tutorial/from-zero-to-hero-write-your-first-kubernetes-charm/create-a-minimal-kubernetes-charm.md b/docs/tutorial/from-zero-to-hero-write-your-first-kubernetes-charm/create-a-minimal-kubernetes-charm.md index b72f6898c..c595a6184 100644 --- a/docs/tutorial/from-zero-to-hero-write-your-first-kubernetes-charm/create-a-minimal-kubernetes-charm.md +++ b/docs/tutorial/from-zero-to-hero-write-your-first-kubernetes-charm/create-a-minimal-kubernetes-charm.md @@ -36,7 +36,7 @@ Conceptually, a charm is code that instructs Juju to deploy and manage an applic All subsequent workload management happens in the same way -- the Juju controller sends events to the charm and the charm responds to these events by managing the workload application in various ways via Pebble. The picture below illustrates all of this for a simple case where there is just one workload container. -![Create a minimal Kubernetes charm](../../resources/create_a_minimal_kubernetes_charm.jpg) +![Create a minimal Kubernetes charm](../../resources/create_a_minimal_kubernetes_charm.png) As a charm developer, your first job is to use this knowledge to create the basic structure and content for your charm: @@ -159,7 +159,7 @@ framework.observe(self.on.demo_server_pebble_ready, self._on_demo_server_pebble_ ```{tip} -**Pro tip:** Use `__init__` to hold references (pointers) to other `Object`s or immutable state only. That is because a charm is reinitialised on every event. See [Talking to a workload: Control flow from A to Z](). +**Pro tip:** Use `__init__` to hold references (pointers) to other `Object`s or immutable state only. That is because a charm is reinitialised on every event. ``` @@ -338,7 +338,7 @@ This name might vary slightly, depending on your architecture. E.g., for an `arm Deploy the `.charm` file, as below. Juju will create a Kubernetes `StatefulSet` named after your application with one replica. -``` +```text juju deploy ./demo-api-charm_ubuntu-22.04-amd64.charm --resource \ demo-server-image=ghcr.io/canonical/api_demo_server:1.0.1 ``` @@ -353,7 +353,7 @@ juju deploy ./demo-api-charm_ubuntu-22.04-amd64.charm --resource \ Monitor your deployment: -``` +```text juju status --watch 1s ``` @@ -382,25 +382,27 @@ You should see a JSON string with the version of the application: {"version":"1.0.0"} ``` ----- + ```{dropdown} Expand if you wish to inspect your deployment further 1. Run: - ``` + +```text kubectl get namespaces ``` + You should see that Juju has created a namespace called `welcome-k8s`. 2. Try: -``` +```text kubectl -n welcome-k8s get pods ``` You should see that your application has been deployed in a pod that has 2 containers running in it, one for the charm and one for the application. The containers talk to each other via the Pebble API using the UNIX socket. -``` +```text NAME READY STATUS RESTARTS AGE modeloperator-5df6588d89-ghxtz 1/1 Running 3 (7d2h ago) 13d demo-api-charm-0 2/2 Running 0 7d2h @@ -408,17 +410,11 @@ demo-api-charm-0 2/2 Running 0 7d2h 3. Check also: -``` +```text kubectl -n welcome-k8s describe pod demo-api-charm-0 ``` In the output you should see the definition for both containers. You'll be able to verify that the default command and arguments for our application container (`demo-server`) have been displaced by the Pebble service. You should be able to verify the same for the charm container (`charm`). - -``` - ------------- - - **Congratulations, you've successfully created a minimal Kubernetes charm!** diff --git a/docs/tutorial/from-zero-to-hero-write-your-first-kubernetes-charm/index.md b/docs/tutorial/from-zero-to-hero-write-your-first-kubernetes-charm/index.md index 623f1d785..e44dfb485 100644 --- a/docs/tutorial/from-zero-to-hero-write-your-first-kubernetes-charm/index.md +++ b/docs/tutorial/from-zero-to-hero-write-your-first-kubernetes-charm/index.md @@ -1,7 +1,7 @@ (from-zero-to-hero-write-your-first-kubernetes-charm)= # From zero to hero: Write your first Kubernetes charm -This tutorial will introduce you to the official way to write a Kubernetes charm --- that is, how to equip an application with all the operational logic that it needs so that you can manage it on any Kubernetes cloud with just a few commands, using [Juju](https://juju.is/docs/olm). +This tutorial will introduce you to the official way to write a Kubernetes charm --- that is, how to equip an application with all the operational logic that it needs so that you can manage it on any Kubernetes cloud with just a few commands, using Juju. ```{important} @@ -23,9 +23,9 @@ The application that we will charm in this tutorial is based on the Python Fast **What you'll need:** - A working station, e.g., a laptop with amd64 architecture. -- Familiarity with [Juju](https://juju.is/docs/juju) -- Familiarity with the Python programming language, Object-Oriented Programming, Event Handlers -- Understanding of Kubernetes fundamentals +- Familiarity with Juju. +- Familiarity with the Python programming language, Object-Oriented Programming, event handlers. +- Understanding of Kubernetes fundamentals. **What you'll do:** @@ -77,13 +77,8 @@ By the end of this tutorial you will have built a machine charm and evolved it i | If you are wondering... | visit... | |-------------------------|----------------------| -| "How do I...?" | {ref}`howto-guides` | +| "How do I...?" | {ref}`how-to-guides` | | "What is...?" | {ref}`reference` | | "Why...?", "So what?" | {ref}`explanation` | - - -
- -> **Contributors:** @beliaev-maksim diff --git a/docs/tutorial/from-zero-to-hero-write-your-first-kubernetes-charm/observe-your-charm-with-cos-lite.md b/docs/tutorial/from-zero-to-hero-write-your-first-kubernetes-charm/observe-your-charm-with-cos-lite.md index 0dd3c8f1f..90f8345ca 100644 --- a/docs/tutorial/from-zero-to-hero-write-your-first-kubernetes-charm/observe-your-charm-with-cos-lite.md +++ b/docs/tutorial/from-zero-to-hero-write-your-first-kubernetes-charm/observe-your-charm-with-cos-lite.md @@ -449,7 +449,8 @@ In a while you should see the following data appearing on the dashboard: 3. Logs from your application that were collected by Loki and forwarded to Grafana. Here you can see some INFO level logs and ERROR logs with traceback from Python when you were calling the `/error` path. -![Observe your charm with COS Lite](../../resources/observe_your_charm_with_cos_lite.png) +![Observe your charm with COS Lite](../../resources/observe_your_charm_with_cos_lite.png) + ```{important} diff --git a/docs/tutorial/from-zero-to-hero-write-your-first-kubernetes-charm/preserve-your-charms-data.md b/docs/tutorial/from-zero-to-hero-write-your-first-kubernetes-charm/preserve-your-charms-data.md index 6d2f14d2b..49dc7a924 100644 --- a/docs/tutorial/from-zero-to-hero-write-your-first-kubernetes-charm/preserve-your-charms-data.md +++ b/docs/tutorial/from-zero-to-hero-write-your-first-kubernetes-charm/preserve-your-charms-data.md @@ -28,15 +28,21 @@ There are a few strategies you can adopt here: First, you can use an Ops construct called `Stored State`. With this strategy you can store data on the local unit (at least, so long as your `main` function doesn't set `use_juju_for_storage` to `True`). However, if your Kubernetes pod dies, your unit also dies, and thus also the data. For this reason this strategy is generally not recommended. -> Read more: [StoredState](), {ref}`StoredState: Uses, Limitations ` +> Read more: [`ops.StoredState`](https://ops.readthedocs.io/en/latest/#ops.StoredState), {ref}`StoredState: Uses, Limitations ` Second, you can make use of the Juju notion of 'peer relations' and 'data bags' and set up a peer relation data bag. This will help you store the information in the Juju's database backend. -> Read more: [Peer integrations](https://juju.is/docs/juju/relation#heading--peer) + + Third, when you have confidential data, you can use Juju secrets (from Juju 3.1 onwards). + + In this chapter we will adopt the second strategy, that is, we will store charm data in a peer relation databag. (We will explore the third strategy in a different scenario in the next chapter.) We will illustrate this strategy with an artificial example where we save the counter of how many times the application pod has been restarted. @@ -49,7 +55,10 @@ peers: fastapi-peer: interface: fastapi_demo_peers ``` + + ## Set and get data from the peer relation databag diff --git a/docs/tutorial/from-zero-to-hero-write-your-first-kubernetes-charm/publish-your-charm-on-charmhub.md b/docs/tutorial/from-zero-to-hero-write-your-first-kubernetes-charm/publish-your-charm-on-charmhub.md index 74f76f35e..7d49aafe6 100644 --- a/docs/tutorial/from-zero-to-hero-write-your-first-kubernetes-charm/publish-your-charm-on-charmhub.md +++ b/docs/tutorial/from-zero-to-hero-write-your-first-kubernetes-charm/publish-your-charm-on-charmhub.md @@ -82,7 +82,7 @@ echo "Random 8-digit hash: $random_hash" ``` ```{important} -Naming your charm is usually less random than that; see [Charm naming guidelines](). However, here we are in a tutorial setting, so you just need to make sure to pick a unique name, any name. +Naming your charm is usually less random than that. However, here we are in a tutorial setting, so you just need to make sure to pick a unique name, any name. ``` @@ -161,7 +161,7 @@ Release your charm as below. ```{important} **Do not worry:**
-While releasing a charm to Charmhub gives it a public URL, the charm will not appear in the Charmhub search results until it has passed formal review -- see [Requirements for public listing](). +While releasing a charm to Charmhub gives it a public URL, the charm will not appear in the Charmhub search results until it has passed formal review. ``` diff --git a/docs/tutorial/write-your-first-machine-charm.md b/docs/tutorial/write-your-first-machine-charm.md index 1bd48dda5..f3c8588df 100644 --- a/docs/tutorial/write-your-first-machine-charm.md +++ b/docs/tutorial/write-your-first-machine-charm.md @@ -1,18 +1,18 @@ (write-your-first-machine-charm)= # Write your first machine charm -In this tutorial you will write a [machine charm](https://juju.is/docs/juju/charmed-operator) for [`juju`](https://juju.is/docs/juju) using (`charmcraft` and) `ops`. +In this tutorial you will write a [machine charm](https://juju.is/docs/juju/charmed-operator) for Juju using (Charmcraft and) Ops. **What you'll need:** - A workstation, e.g., a laptop, with amd64 architecture and which has sufficient resources to launch a virtual machine with 4 CPUs, 8 GB RAM, and 50 GB disk space - Familiarity with Linux -- Familiarity with [Juju](https://juju.is/docs/juju) +- Familiarity with Juju. - Familiarity with object-oriented programming in Python **What you'll do:** -Study your application. Use `charmcraft` and `ops` to build a basic charm and test-deploy it with `juju` and a localhost LXD-based cloud. Repeat the steps to evolve the charm so it can become increasingly more sophisticated. +Study your application. Use Charmcraft and Ops to build a basic charm and test-deploy it with Juju and a localhost LXD-based cloud. Repeat the steps to evolve the charm so it can become increasingly more sophisticated. @@ -35,7 +35,8 @@ The application has other features that we can exploit, but for now this is enou ## Set up your development environment -> See [`juju` | Set up your development environment automatically](https://juju.is/docs/juju/set-up--tear-down-your-test-environment#set-up-automatically) for instructions on how to set up your development environment so that it's ready for you to test-deploy your charm. At the charm directory step, call it `microsample-vm`. At the cloud step, choose LXD. + +> See [Juju | Set up your development environment automatically](https://juju.is/docs/juju/set-up--tear-down-your-test-environment#set-up-automatically) for instructions on how to set up your development environment so that it's ready for you to test-deploy your charm. At the charm directory step, call it `microsample-vm`. At the cloud step, choose LXD. ```{important} @@ -94,7 +95,9 @@ test_charm.py ``` + In your local editor, open the `charmcraft.yaml` file and customise its contents as below (you only have to edit the `title`, `summary`, and `description`): @@ -132,8 +135,9 @@ bases: channel: "22.04" ``` - + Now open the `src/charm.py` file and update it as below (you'll have to add an import statement for `os` and an observer and handler for the `install` event -- in the definition of which you'll be using `os` and `ops`). @@ -167,7 +171,9 @@ if __name__ == "__main__": # pragma: nocover ops.main(MicrosampleVmCharm) # type: ignore ``` + Next, in your Multipass VM shell, inside your project directory, run `charmcraft pack` to pack the charm. It may take a few minutes the first time around but, when it's done, your charm project should contain a `.charm` file. Sample session: @@ -186,7 +192,9 @@ LICENSE microsample-vm_ubuntu-22.04-amd64.charm src README.md pyproject.toml tests ``` + Now, open a new shell into your Multipass VM and use it to configure the Juju log verbosity levels and to start a live debug session: @@ -281,7 +289,9 @@ config: type: string ``` + Then, in the `src/charm.py` file, update the `_on_install` function to make use of the new configuration option, as below: @@ -401,7 +411,9 @@ requests==2.28.1 requests-unixsocket==0.3.0 ``` -> See more: [`charmcraft` | File `requirements.txt` `](), [PyPI > Library `requests`](https://pypi.org/project/requests/), [PyPI > Library `requests-unixsocket`](https://pypi.org/project/requests-unixsocket/) + Then, in your `src/charm.py` file, import the `requests_unixsocket` package, update the `_on_config_changed` function to set the workload version to the output of a function `_getWorkloadVersion`, and define the function to retrieve the Microsample workload version from the `snapd` API via a Unix socket, as below: @@ -510,9 +522,12 @@ Machine State Address Inst id Base AZ Message Congratulations, your charm user can view the version of the workload deployed from your charm! + ## Tear things down -> See [`juju` | Tear down your development environment automatically](https://juju.is/docs/juju/set-up--tear-down-your-test-environment#tear-down-automatically) +> See [Juju | Tear down your development environment automatically](https://juju.is/docs/juju/set-up--tear-down-your-test-environment#tear-down-automatically) + + (tutorial-machines-next-steps)= ## Next steps @@ -521,7 +536,7 @@ By the end of this tutorial you will have built a machine charm and evolved it i | If you are wondering... | visit... | |-------------------------|----------------------| -| "How do I...?" | {ref}`howto-guides` | +| "How do I...?" | {ref}`how-to-guides` | | "What is...?" | {ref}`reference` | | "Why...?", "So what?" | {ref}`explanation` | From 3acafac0c446f4162f08c6213f981e1f5806ae09 Mon Sep 17 00:00:00 2001 From: Tony Meyer Date: Wed, 18 Dec 2024 09:59:48 +1300 Subject: [PATCH 13/16] Leave the suppressed warnings list in place, but empty. --- docs/custom_conf.py | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/docs/custom_conf.py b/docs/custom_conf.py index 109c2ed8f..73aece732 100644 --- a/docs/custom_conf.py +++ b/docs/custom_conf.py @@ -57,11 +57,7 @@ # features, it is sufficient to update the settings in the "Project # information" section. -# FIXME: temporarily disable missing xrefs in this branch, until the missing -# links to Juju and Charmcraft are added. -#suppress_warnings = [ -# 'myst.xref_missing', -#] +suppress_warnings = [] ############################################################ ### Project information From c7c4af176f93b62e1ed83c37f38daa1b76d524bd Mon Sep 17 00:00:00 2001 From: Tony Meyer Date: Wed, 18 Dec 2024 10:03:44 +1300 Subject: [PATCH 14/16] Update the doc locations. --- test/test_infra.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/test_infra.py b/test/test_infra.py index dc4f20761..0e4ac0ced 100644 --- a/test/test_infra.py +++ b/test/test_infra.py @@ -77,8 +77,8 @@ def test_ops_testing_doc(): found_names: typing.Set[str] = set() for test_doc in ( - 'docs/reference/api/ops-testing-harness.rst', - 'docs/reference/api/ops-testing.rst', + 'docs/reference/ops-testing-harness.rst', + 'docs/reference/ops-testing.rst', ): with open(test_doc) as testing_doc: found_names.update({ From a1d6a9b0710424792b8616203040e0d032aeddfc Mon Sep 17 00:00:00 2001 From: Tony Meyer Date: Wed, 18 Dec 2024 10:04:40 +1300 Subject: [PATCH 15/16] Remove unnecessary change. --- docs/custom_conf.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/docs/custom_conf.py b/docs/custom_conf.py index 73aece732..2880c6cae 100644 --- a/docs/custom_conf.py +++ b/docs/custom_conf.py @@ -57,8 +57,6 @@ # features, it is sufficient to update the settings in the "Project # information" section. -suppress_warnings = [] - ############################################################ ### Project information ############################################################ From 8e9dabad7586347940007afdd723ccafe6fda14f Mon Sep 17 00:00:00 2001 From: Dave Wilding Date: Wed, 18 Dec 2024 08:13:08 +0800 Subject: [PATCH 16/16] Fix product name before merging Co-authored-by: Tony Meyer --- docs/index.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/index.md b/docs/index.md index 36acd1cb7..3ca57e1d7 100644 --- a/docs/index.md +++ b/docs/index.md @@ -37,7 +37,7 @@ You can structure your charm however you like, but with the `ops` library, you g :link: tutorial/index :link-type: doc -**Start here**: a hands-on introduction to Example Product for new users +**Start here**: a hands-on introduction to `ops` for new users ``` ```{grid-item-card} [How-to guides](/index)