From 35ac613ab98485f86d69b5d0b38fc34301be9060 Mon Sep 17 00:00:00 2001 From: Dave Morris Date: Tue, 1 Oct 2024 12:54:49 +0100 Subject: [PATCH 1/8] Removed unused note --- notes/zrq/20231002-01-notes.txt | 44 --------------------------------- 1 file changed, 44 deletions(-) delete mode 100644 notes/zrq/20231002-01-notes.txt diff --git a/notes/zrq/20231002-01-notes.txt b/notes/zrq/20231002-01-notes.txt deleted file mode 100644 index 910b0c2..0000000 --- a/notes/zrq/20231002-01-notes.txt +++ /dev/null @@ -1,44 +0,0 @@ -# -# -# -# Copyright (C) 2023 by Wizzard Solutions Ltd, wizzard@metagrid.co.uk -# -# This information is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This information is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . -# -# -# -#zrq-notes-time -#zrq-notes-indent -#zrq-notes-crypto -#zrq-notes-ansible -#zrq-notes-osformat -# -# AIMetrics: [] -# - - Target: - - Misc notes - - Result: - - Work in progress ... - -# ----------------------------------------------------- - - The easiest CI/CD. Ever. - https://buddy.works/ - - - From da6a8630954c86bfa5442d1d35a670775bdb3eb2 Mon Sep 17 00:00:00 2001 From: Dave Morris Date: Tue, 1 Oct 2024 13:55:26 +0100 Subject: [PATCH 2/8] Updated the script to use env file --- notes/zrq/20230916-01-git-branch.txt | 48 +++++++++++++++++++++------- 1 file changed, 36 insertions(+), 12 deletions(-) diff --git a/notes/zrq/20230916-01-git-branch.txt b/notes/zrq/20230916-01-git-branch.txt index 7a44a17..724f6fc 100644 --- a/notes/zrq/20230916-01-git-branch.txt +++ b/notes/zrq/20230916-01-git-branch.txt @@ -29,47 +29,71 @@ Work in progress ... + +# ----------------------------------------------------- +# Create our project env file. +#[user@desktop] + +cat > "${HOME:?}/execbroker.env" << 'EOF' +source "${HOME:?}/projects.env" +EXECBROKER_REPO=git@github.com:Zarquan/ExecutionBroker.git +EXECBROKER_HOME=${PROJECTS_ROOT:?}/IVOA/ivoa-std/ExecutionBroker +EXECBROKER_CODE=${EXECBROKER_HOME:?}/github-zrq +EOF + + # ----------------------------------------------------- # Merge upstream changes. #[user@desktop] - git checkout main + source "${HOME:?}/execbroker.env" + pushd "${EXECBROKER_CODE}" + + git checkout main + + git pull - git pull + git fetch upstream - git fetch upstream + git merge upstream/main - git merge upstream/main + git status - git status + git push - git push + popd # ----------------------------------------------------- # Create a new git branch. #[user@desktop] - branchname=gardening + source "${HOME:?}/execbroker.env" + pushd "${EXECBROKER_CODE}" - newbranch=$(date '+%Y%m%d')-zrq-${branchname:?} + branchname=gardening - git checkout main + newbranch=$(date '+%Y%m%d')-zrq-${branchname:?} - git checkout -b "${newbranch:?}" + git checkout main - git push --set-upstream 'origin' "$(git branch --show-current)" + git checkout -b "${newbranch:?}" + git push --set-upstream 'origin' "$(git branch --show-current)" + + popd # ----------------------------------------------------- # Start a new build container. #[user@desktop] + source "${HOME:?}/execbroker.env" + podman run \ --rm \ --tty \ --interactive \ --name document-builder \ - --volume "$(pwd):/document:rw,z" \ + --volume "${EXECBROKER_CODE}:/document:rw,z" \ debian \ bash From 655833f7180968a9a62b9d51c0296b81db0bd958 Mon Sep 17 00:00:00 2001 From: Dave Morris Date: Tue, 1 Oct 2024 13:56:06 +0100 Subject: [PATCH 3/8] Initial plan for purls --- notes/zrq/20241001-01-github-urls.txt | 54 +++++++++++++++++++++++++++ 1 file changed, 54 insertions(+) create mode 100644 notes/zrq/20241001-01-github-urls.txt diff --git a/notes/zrq/20241001-01-github-urls.txt b/notes/zrq/20241001-01-github-urls.txt new file mode 100644 index 0000000..de75a9e --- /dev/null +++ b/notes/zrq/20241001-01-github-urls.txt @@ -0,0 +1,54 @@ +# +# +# +# Copyright (c) 2024, Manchester (http://www.manchester.ac.uk/) +# +# This information is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This information is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +# +# +#zrq-notes-indent +# +# AIMetrics: [] +# + + Target: + + Replace purls with github.io urls in the text. + Add purls to link to them as a test. + + Result: + + Work in progress ... + This is probably a different branch ? + +# ----------------------------------------------------- + + # + # Login to archive.org using my own identity. + # + # Login to purl.archive.org to manage our URLs + # https://purl.archive.org/domain/ivoa.net + # + # Not sure how reliable this is at the moment. + # Create everything as github.io urls first. + # + +# ----------------------------------------------------- + + + # + # Add a core set of URLs for our core types. + # + From f772a327e27b2ee3c48282a79e3fe02741366913 Mon Sep 17 00:00:00 2001 From: Dave Morris Date: Tue, 8 Oct 2024 17:45:03 +0100 Subject: [PATCH 4/8] Transfer from laptop to desktop .. --- ExecutionBroker.tex | 695 ++++++++++++++++++++++++++------------------ 1 file changed, 408 insertions(+), 287 deletions(-) diff --git a/ExecutionBroker.tex b/ExecutionBroker.tex index a29bcc3..6dbd03e 100644 --- a/ExecutionBroker.tex +++ b/ExecutionBroker.tex @@ -13,20 +13,38 @@ \newcommand{\json} {JSON} \newcommand{\yaml} {YAML} \newcommand{\http} {HTTP} +\newcommand{\rest} {REST} \newcommand{\datamodel} {data~model} \newcommand{\webservice} {web service} \newcommand{\webbrowser} {web browser} +\newcommand{\vo} {VO} +\newcommand{\vofull} {Virtual Observatory} \newcommand{\ivoa} {IVOA} +\newcommand{\ivoafull} {International Virtual Observatory Alliance} \newcommand{\uws} {UWS} \newcommand{\vospace} {VOSpace} +\newcommand{\execworkerclass} {**ExecutionWorker**} \newcommand{\execbrokerclass} {ExecutionBroker} -\newcommand{\execworkerclass} {ExecutionWorker} +\newcommand{\execbrokerservice}[1] {ExecutionBroker~service#1} + +\newcommand{\execoffer}[1] {\textit{ExecutionBroker~offer#1}} +\newcommand{\execofferset}[1] {\textit{ExecutionBroker~offerset#1}} +\newcommand{\execsession}[1] {\textit{ExecutionBroker~session#1}} + \newcommand{\executionbroker} {Execution~Broker} \newcommand{\executionplanning} {Execution~Planning} +\newcommand{\executable} {\textit{executable}} +\newcommand{\executablething}[1] {\textit{executable~thing#1}} +\newcommand{\excutabletask} {\textit{executable} task} + +%\newcommand{\execoffer}[1] {\textit{offer#1}} +\newcommand{\workerjob}[1] {\textit{session#1}} +\newcommand{\teardown} {tear-down} + \newcommand{\jupyter} {Jupyter} \newcommand{\jupyterhub} {JupyterHub} \newcommand{\binderhub} {BinderHub} @@ -67,14 +85,6 @@ \newcommand{\datascience} {data~science} \newcommand{\scienceplatform}[1] {science~platform#1} -\newcommand{\executable} {\textit{executable}} -\newcommand{\executablething}[1] {\textit{executable~thing#1}} -\newcommand{\excutabletask} {\textit{executable} task} - -\newcommand{\execoffer}[1] {\textit{offer#1}} -\newcommand{\workerjob}[1] {\textit{session#1}} -\newcommand{\teardown} {tear-down} - \newcommand{\cpu}[1] {CPU#1} \newcommand{\gpu}[1] {GPU#1} \newcommand{\nvidiagpu} {NVIDIA~AD104~GPU} @@ -165,7 +175,7 @@ \begin{abstract} \label{abstract} -One of the long term goals of the IVOA has been to enable users to +One of the long term goals of the \ivoa{} has been to enable users to move the code to the data. This is becoming more and more important as the size and complexity of the \dataset{s} available in the virtual observatory increases. @@ -215,15 +225,14 @@ \section{Introduction} \label{introduction} The \ivoa{} \executionbroker{} specification defines a \datamodel{} for describing executable tasks -and two \webservice{} interfaces, \execbrokerclass{} and \execworkerclass{}. +and a \webservice{} interface for managing them. Together these provide a common interface for service discovery, resource allocation and execution scheduling across a heterogeneous federation of different types of execution platform. \begin{itemize} \item \execbrokerclass{} \datamodel{} – a data model for describing execution sessions and their resource requirements. - \item \execbrokerclass{} \webservice{} – a discovery service to find execution platforms, allocate resources and schedule execution sessions. - \item \execworkerclass{} \webservice{} – a monitoring service for monitoring and interacting with execution sessions. + \item \execbrokerclass{} \webservice{} – a \rest{} based web service to find execution platforms, allocate resources and schedule execution sessions. \end{itemize} \subsection{Role within the VO Architecture} @@ -241,7 +250,7 @@ \subsection{Role within the VO Architecture} \label{fig:archdiag} \end{figure} -The IVOA Architecture\citep{2010ivoa.rept.1123A} provides a high-level view of how IVOA +The \ivoa{} Architecture\citep{2010ivoa.rept.1123A} provides a high-level view of how \ivoa{} standards work together to connect users and applications with providers of data and services. Fig.~\ref{fig:archdiag} shows the role the \ivoa{} \executionbroker{} plays within this architecture. @@ -251,13 +260,13 @@ \subsection{Role within the VO Architecture} together the \dataset{s} co-located with the compute resources needed to analyse them.\footurl{https://data.lsst.cloud/}\footurl{https://rsp.lsst.io/index.html} -The \scienceplatform{s} make extensive use of the \ivoa{} data models and +These \scienceplatform{s} make extensive use of the \ivoa{} data models and vocabularies to describe their \dataset{s}, and use the \ivoa{} data access services to find and access data from other data providers. In addition, some of the \scienceplatform{s} use \ivoa{} \vospace{} services to manage data transfers to and from local storage co-located with the compute resources. -However, to date the \ivoa{} does not provide any APIs or \webservice{} interfaces that +However, to date the \ivoa{} does not provide any APIs or services that enable \scienceplatform{s} to exchange the software used to analyse the data. The \ivoa{} \executionbroker{} provides a step towards making this possible. @@ -267,18 +276,13 @@ \subsection{Role within the VO Architecture} resource allocation and execution scheduling across a heterogeneous federation of execution platforms. -The \ivoa{} \executionbroker{} specification refers to the +\ivoa{} \executionbroker{} services may use the \ivoa{} Single-Sign-On standard \citep{2017ivoa.spec.0524T} for authentication (see section xx) %\ref{subsec:authentication} and the \ivoa{} Credential Delegation Protocol \citep{2010ivoa.spec.0218P} for delegating credentials to other services. -The \ivoa{} \executionbroker{} specification also describes how to register -an \execbrokerclass{} service in the -\ivoa{} Registry \citep{2009ivoa.spec.1104B}, -making it findable within the wider context of the VO. - \subsection{Executable things} \label{executablething} @@ -299,13 +303,13 @@ \subsection{Executable things} This is the first identifiable \executablething{} in our example. To be able to use this \executablething{}, you would need a computing resource with the appropriate hardware and software environment. In this case, a computing resource with the \python{} interpreter -installed along with any additional \python{} modules required by the program. +installed along with the additional \python{} modules required by the program. This environment is often referred to as the \python{} runtime. In the context of \scienceplatform{s} and \datascience{}, a common pattern is to provide this environment using a Docker\footurl{https://docs.docker.com/get-started/what-is-a-container/} or OCI\footurl{https://opencontainers.org/} container -to package the \pythonprogram{} and runtime together as a single binary object. +to package the \pythonprogram{} and \python{} runtime together as a single binary object. This package, or container, is itself an \executablething{}. One which requires a different execution environment than the original \pythonprogram{}. The aim of containerization is to package software components together with all the libraries and dependencies @@ -342,20 +346,20 @@ \subsection{Executable things} are and what resources are needed to execute them. This can include things like number of \cpu{} cores and amount of memory it needs, whether it needs a \gpu{}, the location of the input data, the storage space needed to perform -the calculation and the storage space needed to save the results. +the calculation, and the storage space needed to save the results. \section{Service interaction} \label{service-interaction} -The interaction between user, client and services can be described as a conversation between the client -and one or more \execbrokerclass{} services to discover where, how, and when, an \executablething{} can be -executed. +The interaction between a user, the client application they are using, and the services available in the \vo{} +can be described as a conversation between the client and one or more \execbrokerservice{s} to discover +where, how, and when, an \executablething{} can be executed. \subsection{Discovery services} \label{discovery-services} The conversation starts at the discovery stage, where the user uses discovery services to -selects the software and \dataset{s} that they want to work with. +select the software and \dataset{s} that they want to work with. \includegraphics[width=0.9\textwidth]{diagrams/data-discovery.pdf} @@ -371,40 +375,222 @@ \subsection{Discovery services} coordinate system. The programming language the software is written in and the file format of the \dataset{} are at best secondary criteria. +In our square root example, we would expect our user to use search terms like \textit{'square root'} +or \textit{'newton raphson'} to find the software they need. +We wouldn't expect them to start out looking for a \textit{'python'} program or a \textit{'docker'} container. Ideally, if the \executionbroker{} services function as intended, a science user should not -need to know about programming languages or file formats. +need to know about programming languages, software packaging or file formats. The \executionbroker{} services should make all of the technical details invisible, enabling the science user to get on with science. -The interfaces for the discovery services should be designed to be domain agnostic. -Meaning that it should be possible to swap out the astronomy based discovery services -for the equivalent biochemistry discovery services and although the domain specific +Another important consideration is these discovery services should be designed to be domain agnostic. +Meaning that it should be possible to swap out an astronomy based discovery service +for an equivalent biochemistry discovery service and although the domain specific terms and vocabulary will be different, the techical details of the service interfaces should be the same. -\subsubsection{Separation of concerns} -\label{separate-concerns} -Separate who knows what .. + +%move this to a different section ... +\subsection{Metadata roles} +\label{metadata-roles} + +The full description of an \executablething{} consists several layers of metadata, +each of which may be provided by different actors playing different roles within the community. + +For our square root \pythonprogram{} example we can identify a number of roles that each provide +part of the picture nedeed to fully describe the task. + The players: \begin{itemize} - \item The researcher who creates the notebook - \item The developer who creates the container - \item The publisher who publishes the data - \item The user who is running the analysis + \item The developer - The person who wrote the program + \item The packager - The person who packaged it in a container + \item The data provider - The data provider who publishes the data + \item The user - The person who is running the analysis \end{itemize} +\subsubsection{The software developer} +\label{software-developer} + +The first layer of metadata comes from the person who wrote the \pythonprogram{}. +They have detailed knowledge of what the software does, what execution environment it needs, +and what the inputs and outputs are. + +For the square root example, it is a \pythonprogram{} which needs a platform with the \python{} runtime installed, +and a list of the \python{} libraries that the program relies on. + +\begin{lstlisting}[] +# ExecutionBroker service response. +executable: + type: uri:python-program + requirements: + - numpi: "" + - astropy: ">= 6.1" + +\end{lstlisting} + +The developer also understands how much memory their program needs, whether it can make use of multiple cpu cores, +and whether it can make use of a \gpu accelerator. + +So they can provide an initial description of the compute resources +their program requires. + +\begin{lstlisting}[] +compute: +- type: uri:generic-compute + cores: + memory: + extras: + - type: uri:gpu-accellerator + .... +\end{lstlisting} + +They also know what inputs the program expects to receive and how it outputs the result. +If it is being used to process a list of numbers, the developer knows where their program expects to find the input data, +what file formats can it accept for the input, where it puts the results, and what output formats can it generate. + +\begin{lstlisting}[] +compute: + volumes: + - path: /... + mode: readonly + resource: input data +\end{lstlisting} + +\begin{lstlisting}[] +data: +- type: uri:data-descriptor + name: input data + mode: readonly + data-type: List + data-formats: + - type: uri:dataformat-csv + .... + - type: uri:dataformat-tsv + .... + - type: uri:votable + .... +- type: uri:data-descriptor + name: output data + mode: readwrite + data-type: List + data-formats: + - type: uri:dataformat-csv + .... + - type: uri:dataformat-tsv + .... + - type: uri:votable + .... +\end{lstlisting} + +ZRQ +Question - what is the relationship between data-descriptor and data-resource ? +Are they parts of the same thing, or are they two different things ? +Do we replace the data-descriptor with a data-resource, or do we just extend it ? + +\subsubsection{The software packager} +\label{software-packager} + +Packager changes the type of \executablething{} from a \pythonprogram{} +to a \dockercontainer{}. + +\begin{lstlisting}[] +executable: + type: uri:docker-container + .... +\end{lstlisting} + +Depending on how they package the software they will probably need to change the +description of where the \executablething{} expects to find its +inputs and outputs. + +\begin{lstlisting}[] +compute: + volumes: + - .... +\end{lstlisting} + +\subsubsection{The data provider} +\label{data-provider} + +The data provider describes what the data contains and what formats it is available in. +This can be used to match up data sets with input and outputs of an \executablething{}. + +Beyond the scope of this specification, the schemas and vocabularies needed for this +level of metadata is the domain of the semantics working group. + +At this stage it is probably not practical to try to define a universal set of data +structures for all of the data across all of the \ivoa{}. + +However, it may be possible to start with an initial set of data types for +individual projects and gradually work towards a wider concensus. + +Examples + + uri:eMerlin-visibility-data-001 + uri:lofar-visibility-data-001 + uri:lofar-visibility-data-001 + ... + +Even just applying a URI to identify the data format will enable +basic tools to identify which data can be used with which software. + +\subsubsection{The user} +\label{user-input} + +The user provides the last step in the metadata. +By selecting the data they want to use as inouts, and selecting what format they want the output +to be in, they can refine the links between the +data resources, data descriptions and volume mounts. + +In most cases the user will use the +compute resource requiremenst set by the software developer, +but they can provide extra information +based on their knowledge of what they plan to use the software for. + +If the original software was packaged as an example notebook, with +fairly low requirements for compute resources, but the user intends to use the software +to analyse a much larger data set they can modify the compute resource requirements +to match their use case. + +\begin{lstlisting}[] +compute: +- type: uri:generic-compute + cores: + memory: +\end{lstlisting} + + + \subsection{Execution broker} -\label{execution-broker-desc} +\label{execution-broker-intro} Once the user has selected the \executablething{} they want to use and the data they want to apply it to, the client combines this information to create a -description of the session the user wants to execute and sends it to one or more -\execbrokerclass{} services. -Each \execbrokerclass{} service evaluates the request and responds with a top level +complete description of the \execsession{} the user wants to execute, including +details of the executable, the compute, storage, and data resources it needs, +and a schedule describing when the user wants it to run. + +\begin{lstlisting}[] +executable: + .... +resources: + .... +schedule: + .... +\end{lstlisting} + +\subsubsection{Request for offers} +\label{offers-request} + +The client sends the \execsession{} description to one or more \execbrokerclass{} +services to see if they can meet the requirements. + +Each \execbrokerservice{} evaluates the request and responds with a top level \codeword{YES|NO} answer, and if the answer is \codeword{YES}, a list of one or more \execoffer{s} describing how -the requested session could be executed on that platform. +the requested \execsession{} could be executed on the platform(s) represented by +that \execbrokerservice{}. %\begin{lstlisting}[] %Request - Can this platform execute ? @@ -413,58 +599,128 @@ \subsection{Execution broker} \includegraphics[width=0.9\textwidth]{diagrams/request-offers.pdf} -Each \execoffer{} contains some metadata about the \execoffer{} itself, -including an identifier and an expiry time, -followed by a description of the \workerjob{} -including details of when and how it would be executed. +\subsubsection{Offerset response} +\label{offerset-response} + +The \execbrokerservice{} will respond to a request for offers with an \execofferset{} +containing some metadata about the \execofferset{} itself, and a list of \execoffer{}s +describing how the requested session could be executed. + +Each \execoffer{} in the list contains some metadata about the \execoffer{} itself, +including it's identifier and expiry time, followed by details of how and when the +\execsession{} would be executed. \begin{lstlisting}[] -# ExecutionBroker service response. -response: - result: 'YES' - offers: - - ident: "f185cd47-db9b-4765-a934-bf81ee9d2d34" - status: 'OFFERED' - expires: "2023-09-18T07:05:21" - body: - executable: - .... - resources: - .... - datetime: - .... - - ident: "2c89f536-3fff-48f7-943f-bcc5c3225be7" - status: 'OFFERED' - expires: "2023-09-18T07:05:21" - body: - executable: - .... - resources: - .... - datetime: - .... +# ExecutionBroker offerset response. +result: 'YES' +state: 'OFFERED' +expires: "2023-09-18T07:05:21" +.... +offers: + +- ident: "2e164a1b-7ff6-11ef-8412-4bc36fe2face" + state: 'OFFERED' + expires: "2023-09-18T07:05:21" + .... + executable: + .... + resources: + .... + schedule: + .... + +- ident: "2e16bf4c-7ff6-11ef-8412-4bc36fe2face" + status: 'OFFERED' + expires: "2023-09-18T07:05:21" + .... + executable: + .... + resources: + .... + schedule: + .... \end{lstlisting} -The client can then ask its user to choose which of the \execoffer{s} best fits their requirements -and sends a message to the \execbrokerclass{} accepting that offer. +The user can choose to accept one of the \execoffer{s} from the list that best +fits their requirements, or they can reject all of the \execoffer{s} in the +\execofferset{} and make a new request with different criteria. + +Each of the \execoffer{s} in the \execofferset{} represent a temporary reservation +for the resources listed in the \execoffer{}. +This means these resources will not available to other users while the \execoffer{s} +are still valid. + +If the user does nothing, then the \execbrokerservice{} will automatically +update the \codeword{state} of each of the \execoffer{s} to \codeword{EXPIRED} +when their expiry time is reached and release the resources associated with them. + +If the user accepts one of the \execoffer{s} in the \execofferset{} by updating +the \codeword{state} to \codeword{ACCEPTED}, the \execbrokerservice{} SHOULD +update the \codeword{state} of the other \execoffer{s} in the \execofferset{} to +\codeword{REJECTED} and release the associated resources. + +[accept/reject/expire state-transition diagram ?] + +\subsubsection{Execution state} +\label{execution-state} + +Once the offer of anf \execsession{} has been accepted, the \execbrokerservice{} +will begin the process of executing it. + +Each \execsession{} has a unique URL that the client can use to monitor and update +its state. + +The \execbrokerservice{} response for an \execsession{} includes a list of options +that the user may to perform on the \execsession{}. +Which options are available will depend on the current \codeword{state} of the +\execsession{} and the identity and permissions of the authenticated user. + +If the \execoffer{} is still being offered, then the list of available options +allow the user to accept or reject the offer by updating the \codeword{state} of +the \execsession{} to \codeword{ACCEPTED} or \codeword{REJECTED}. + +\begin{lstlisting}[] +ident: "2e164a1b-7ff6-11ef-8412-4bc36fe2face" +state: 'OFFERED' +expires: "2023-09-18T07:05:21" +.... +.... +options: + - type: "urn:enum-value-option" + path: "state" + values: + - "ACCEPTED" + - "REJECTED" +\end{lstlisting} + +Once the \execoffer{} has been accepted and the \execsession{} has started to +execute, then the list of available options will only allow the user to cancel +the execution. + +\begin{lstlisting}[] +ident: "2e164a1b-7ff6-11ef-8412-4bc36fe2face" +state: 'ACCEPTED' +expires: "2023-09-18T07:05:21" +.... +.... +options: + - type: "urn:enum-value-option" + path: "state" + values: + - "CANCELLED" +\end{lstlisting} + +This pattern of using a dynamic list of options enables an \execbrokerservice{} +to change what an individual user is able to do and communicate that to the +client over the lifetime of an \execsession{}. + + + + -In response, the \execbrokerclass{} will update the status of the \execoffer{} to \codeword{ACCEPTED} and return a URL -pointing to a \workerjob{} in a \execworkerclass{} service, enabling the client to monitor the \workerjob{} -status while it is executing. -%\begin{lstlisting}[] -%Request - accept -%Response - -%\end{lstlisting} -\includegraphics[width=0.9\textwidth]{diagrams/accept-offer.pdf} -Once the client has accepted an \execoffer{}, the conversation between the client and the \execbrokerclass{} -is finished. Once one \execoffer{} has been accepted, the \execbrokerclass{} will cancel the other \execoffer{s} and -release their resources. -The client does not need to cancel the \execoffer{s} made by the other \execbrokerclass{} services. -\execoffer{s} are only valid for a limited period of time and they will expire naturally when they reach -the end of their lifetime. How the \execbrokerclass{} and \execworkerclass{} services are linked is up to the implementation architecture. One implementation may package both service interfaces into a single web-application. @@ -487,6 +743,25 @@ \subsection{Execution worker} A \workerjob{} in an \execworkerclass{} service goes through the following stages in its lifecycle. \begin{itemize} + + + - PROPOSED + + - OFFERED + - ACCEPTED + - REJECTED + - EXPIRED + + - WAITING + - PREPARING + - READY + - RUNNING + - FINISHING + - COMPLETED + - FAILED + - CANCELLED + + \item \codeword{PENDING} The \workerjob{} has been created, but the resources have not be prepared yet. \item \codeword{SETUP} The \execworkerclass{} service is setting up the resources needed to execute the \workerjob{}. This includes things like staging any \dataset{s} that will be needed locally. @@ -1124,14 +1399,14 @@ \subsection{Storage resources} By setting the storage resource type to \codeword{https://.../persistent}, and setting the \codeword{lifecycle} to \codeword{managed}, -the client is asking the \execbrokerclass{} service to take care of allocating +the client is asking the \execbrokerservice{} to take care of allocating the storage and managing its lifecycle. -It is up to the \execbrokerclass{} service to decide where to store the data and +It is up to the \execbrokerservice{} to decide where to store the data and how make it accessible after the \workerjob{} has completed. For example, a science platform may have a \rucio{} storage system co-located with the compute platform which it uses to store user generated data. -In which case the \execbrokerclass{} service would respond with an \execoffer{} that +In which case the \execbrokerservice{} would respond with an \execoffer{} that stores the results in the \rucio{} system and provides details of how the user can access it after the \workerjob{} has completed. @@ -1186,7 +1461,7 @@ \subsection{Storage resources} It is important to note that at this point in time the storage is proposed, but not yet allocated. The persistent storage is only allocated if the client accepts this particular \execoffer{}. -This allows an \execbrokerclass{} service to make multiple \execoffer{s} with different storage options, +This allows an \execbrokerservice{} to make multiple \execoffer{s} with different storage options, allowing the client to select and accept the one that best fits its use case. The same \datamodel{} can be used the other way around as well. @@ -1221,7 +1496,7 @@ \subsection{Storage resources} lifecycle: "unmanaged" \end{lstlisting} -It is up to the \execbrokerclass{} service to work out if it is able to access the +It is up to the \execbrokerservice{} to work out if it is able to access the \vospace{} location and mount it as a volume in the compute resource, using either its own authentication, or a delegated form of the authentication provided by the client. @@ -1233,7 +1508,7 @@ \subsection{Storage resources} Note that in this example, the client has specified the lifecycle as \codeword{unmanaged}, which means that the \execbrokerclass{} is not involved in managing the creation or deletion of the data in \vospace. -It is also possible for the client to ask the \execbrokerclass{} service to manage +It is also possible for the client to ask the \execbrokerservice{} to manage data in an external resource. \begin{lstlisting}[] @@ -1258,164 +1533,27 @@ \subsection{Storage resources} \end{lstlisting} In this example, the client is specifying an external \vospace{} location to store the data, -but it is asking the \execbrokerclass{} service to manage the lifecycle, creating the location +but it is asking the \execbrokerservice{} to manage the lifecycle, creating the location in \vospace{} at the start of the \workerjob{} and deleting it 2 days after the \workerjob{} completes. This negotiation of who is responsible for creating and deleting storage resources enables a client to put together a workflow of interconnected steps, with the -\execbrokerclass{} services +\execbrokerservice{s} managing the lifecycle of the resources and releasing them automatically after they are no longer needed. \section{Authentication} \label{authentication} -The client may also want to check whether the user account making the request -has sufficient access rights and resource quota needed to execute it. - -This is equivalent to asking: -\textit{"Do I have permission to use these resources to run this \jupyternotebook{}?"} +The client / service interactions requires some level of authentication to enable the +\execbrokerservice{} to control access to the -There are two ways for the \execbrokerclass{} service to check the user's identity, -using implicit and explicit authentication. +The \executionbroker{} standard does not mandate a particular authentication method. +However, to facilitate interoperability with as wide a community as possible an individual \execbrokerclass{} +service should follow the guidlines set out in the \ivoa{} SSO standard. -The implicit method is for the client to authenticate to the \execbrokerclass{} service -as normal and the service will use the identity from the request headers to check if the -user has sufficient access rights and resource quota. -If the \webservice{} call to the \execbrokerclass{} service is authenticated -using OpenIDConnect -(OIDC)\footurl{https://auth0.com/docs/authenticate/protocols/openid-connect-protocol}, -then the \execbrokerclass{} MUST include a reference to the implicit -authentication in its reply, including enough non-secret -information to identify the authenticated identity. -\begin{lstlisting}[] -# ExecutionBroker server response. -response: - .... - # Details of the authentication offered. - authentication: - - name: "http-request" - type: "https://.../oidc" - mode: "implicit" - spec: - subject: "user@domain" -\end{lstlisting} - -If the client uses an unknown authentication method in the \webservice{} call and -the \execbrokerclass{} service recognises that an anthentication attempt has been made, -then it MUST reject the \webservice{} call with a 401 "Unauthenticated" response. -If the \execbrokerclass{} service does not realise that an anthentication attempt has been made, -then this MAY result in the request being treated as an anonymous unauthenticated call. - -The \datamodel{} also allows for an explicit statement of identity in the request. -The \datamodel{} for authentication follows the same pattern as previous sections, -defining a \codeword{type} URI to identify the type of authentication method, -and a \codeword{spec} section for the type specific details. - -For example, to explicitly include a basic authentication with username and password -in the request: -\begin{lstlisting}[] -# ExecutionBroker client request. -request: - # Details of the executable. - executable: - .... - # Additional authentication methods - authentication: - - name: "basic-auth" - type: "https://.../basic" - spec: - username: "..." - password: "..." -\end{lstlisting} - -This pattern allows the client to supply multiple authentication methods -in the request. The \execbrokerclass{} service can select the authentication -methods it accepts from those in the request and include the name and -type of the methods in its \execoffer{} along with enough non-secret -information to identify the authenticated identity. - -For example, if the client supplies both basic and token authentication -in the request: -\begin{lstlisting}[] -# ExecutionBroker client request. -request: - # Details of the executable. - executable: - .... - # Additional authentication methods - authentication: - - name: "basic-auth" - type: "https://.../basic" - mode: "explicit" - spec: - username: "..." - password: "..." - - name: "token-auth" - type: "https://.../token" - mode: "explicit" - spec: - token: "..." -\end{lstlisting} - -If the \execbrokerclass{} service only accepts the token authentication, it should -skip the basic authentication and only include a reference to the token -based authentication in its \execoffer{}. - -\begin{lstlisting}[] -# ExecutionBroker server response. -response: - .... - # Details of the authentication offered. - authentication: - - name: "token-auth" - type: "https://.../rfc7519" - mode: "explicit" - spec: - .... -\end{lstlisting} - -If the \execbrokerclass{} service does not recognise or support any of the authentication methods -included in the request, then the service MUST reject the request and reply with \codeword{NO}. - -\begin{lstlisting}[] -Request - Can I authenticate with ? -Response - NO -\end{lstlisting} - -The result is that an \execoffer{} made by an \execbrokerclass{} service should only include details -of the authenticated identities that are allowed to use the \execoffer{}. - -If the original question is equivalent to: -\textit{"Do \textbf{these identities} have sufficient access rights and quota to run this \jupyternotebook{}?"} - -Then the response from the \execbrokerclass{} service is equivalent to: -\textit{"This \execoffer{} asserts that \textbf{these identities} have sufficient access rights and quota to run this \jupyternotebook{}?"} - -The client MUST use one of the authentication methods listed in the \execoffer{} when -contacting the \execbrokerclass{} service to accept the \execoffer{} and start the \workerjob{}. -The client MUST continue to use the same authentication method when making subsequent -requests to the \execworkerclass{} service to access the \workerjob{} status and results. - -The \executionbroker{} specification includes an initial set of authentication methods -corresponding to the methods defined in the -\ivoa{} Single-Sign-On standard\citep{2017ivoa.spec.0524T} - -\begin{itemize} - \item .... - \item .... -\end{itemize} - -However, the \datamodel{} also allows an \execbrokerclass{} service to accept authentication -methods that are not covered by the \ivoa{} SSO standard. -A client is free to use any authentication method, including ones not covered by the -\ivoa{} SSO standard. It is up to the \execbrokerclass{} service to decide how it -handles the authentication infomation it receives. - -This means that the \executionbroker{} services can be deployed in other domains outside the \ivoa{}, -without requiring them to adopt the full \ivoa{} SSO standard. \section{Date and time} \label{date-time} @@ -1426,7 +1564,7 @@ \section{Date and time} The client can specify one or more time periods when it would like to start the \workerjob{}, and the minimum duration that it thinks would be needed to complete the \workerjob{}. -The \execbrokerclass{} service may respond with one or more \execoffer{s} that specify when the \workerjob{} +The \execbrokerservice{} may respond with one or more \execoffer{s} that specify when the \workerjob{} would start and the maximum duration that the \workerjob{} would be allowed to consume. It is then up to the client to select which of the \execoffer{s} best suits its use case. @@ -1468,7 +1606,7 @@ \section{Date and time} - start: "2023-08-14T11:30Z/PT30M" \end{lstlisting} -The \execbrokerclass{} service SHOULD respond with one or more \execoffer{s} that start within +The \execbrokerservice{} SHOULD respond with one or more \execoffer{s} that start within the ranges specified in the request. The start times in the \execoffer{s} MAY be more precise than the start times in the request, but they MUST all occur within at least one of the ranges specified in the request. @@ -1488,7 +1626,7 @@ \section{Date and time} min: "P1H" \end{lstlisting} -The \execbrokerclass{} service MAY respond with \execoffer{s} that start at different times and +The \execbrokerservice{} MAY respond with \execoffer{s} that start at different times and set different values for the maximum duration. It MAY offer a maximum duration of 1 hour in the morning, starting at 11:00. @@ -1533,7 +1671,7 @@ \section{Date and time} min: "P1H" \end{lstlisting} -The \execbrokerclass{} service may respond with 2 \execoffer{s}, +The \execbrokerservice{} may respond with 2 \execoffer{s}, the minimum 2 cores and 2G of memory for 1 hour starting at 11:30, and a larger \execoffer{} of up to 8 cores and 8Gb of memory for up to 4 hours starting between 22:00 and 23:00. @@ -1580,10 +1718,10 @@ \section{Date and time} Accept the limited \execoffer{} in the morning, or accept the more generous \execoffer{} with more resources and more time later in the day. -If an \execbrokerclass{} service is offering more than one option for the \codeword{datetime} +If an \execbrokerservice{} is offering more than one option for the \codeword{datetime} section it MUST make separate \execoffer{s} for each different option. Even if all of the other parameters are the same, e.g. compute and storage resources, the -\execbrokerclass{} service MUST NOT include more than one time slot in the same \execoffer{}. +\execbrokerservice{} MUST NOT include more than one time slot in the same \execoffer{}. Technically the \datamodel{} allows an array of values for the \codeword{datetime} section, but this would impose unnecessary complexity on the client for no real gain in user experience. @@ -1593,9 +1731,9 @@ \section{Date and time} \section{Federated architecture} \label{federation} -The \execbrokerclass{} and \execworkerclass{} services are designed to be used at multiple levels within an organization. +The \execbrokerservice{} is designed to be used at multiple levels within an organization. -At the low level, an \execbrokerclass{} and \execworkerclass{} pair may be implemented as a +At the low level, an \execbrokerclass{} serice may be implemented as a single web-application linked to a simple \docker{} execution service. The configuration may be hard coded to only accept a fixed white list of container images and a fixed allocation of compute resources for each job{}. @@ -1612,7 +1750,7 @@ \section{Federated architecture} This information could come from the \ivoa{} Registry, or it could simply be provided as a flat list in a configuration file for the client. -A more flexible architecture could add another \execbrokerclass{} service +A more flexible architecture could add another \execbrokerservice{} a level above the low level task specific services. This service would be configured with a list of the local task specific services and act as an aggregator service sitting between the client and the low level services. @@ -1629,29 +1767,27 @@ \section{Federated architecture} The aggregator service may also have an understanding of the location of \dataset{s} within the organization and be able to route requests to different low level services depending on which \dataset{s} were required. -The aggregator service may expose the low level \execworkerclass{} endpoints in its responses, -or it may implement a proxy interface acting as an aggregator for the \execworkerclass{} -services as well. +The aggregator service may expose the individual low level \execbrokerclass{} endpoints in its responses, +or it may implement a proxy interface hiding the individual services. -This configuration could be used to provide \execbrokerclass{} and \execworkerclass{} service interfaces -that bridged a firewall. Providing a public interface for external clients and forwarding the requests -to the internal \execbrokerclass{} and \execworkerclass{} services that are not accessible from outside the +This configuration could be used to provide an \execbrokerservice{} interface +that bridges a firewall. Providing a public interface for external clients and forwarding the requests +to internal \execbrokerservice{s} that are not accessible from outside the firewall. -A large organization with multiple sites couls also deploy a single high level \execbrokerclass{} and \execworkerclass{} -interface that handles task execution for the whole organization, forwarding the requests to mid-level -\execbrokerclass{} and \execworkerclass{} services at each site which in turn forwards the requests to -individual low-level \execbrokerclass{} and \execworkerclass{} services within the local site networks. +A large organization with multiple sites could also deploy a single high level \execbrokerclass{} +interface that handles requests for the whole organization, forwarding individual requests to mid-level +\execbrokerservice{s} at each site which in turn forwards the requests to +individual low-level \execbrokerservice{s} for each platform within the local sites. The implementation at each level may be different, providing different levels of routing and aggregation based on internal knowledge of the capabilities of the level below, -but the \execbrokerclass{} and \execworkerclass{} interfaces would be the same at each level -in the organization. +but the \execbrokerclass{} interfaces would all be the same. -This could be expanded to include another level of \execbrokerclass{} and \execworkerclass{} services that +This could be expanded to include another level of \execbrokerservice{s} that cross organization bondaries, providing a gateway that allows users from one project to access services from other projects and organizations. This gateway service may also provide an authentication translation -layer between external public accounts internal project specific identities and authentication methods. +layer between external public accounts and internal project specific identities and authentication methods. Federated cloud diagram .... @@ -1847,11 +1983,11 @@ \subsubsection{Request method} } \end{lstlisting} -The \execbrokerclass{} service will evaluate the request and respond with either a positive +The \execbrokerservice{} will evaluate the request and respond with either a positive or negative \codeword{response} depending on whether the platform is able to execute the task descibed in the \codeword{request}. -An \execbrokerclass{} service MUST be able to provide both \yaml{} and \json{} serializations +An \execbrokerservice{} MUST be able to provide both \yaml{} and \json{} serializations of the \codeword{response}. The serialization is determined by the \codeword{Accept} header in the HTTP request. \begin{itemize} @@ -1919,7 +2055,7 @@ \subsubsection{Offer GET} The \datamodel{} for the content of the response is described in section \ref{datamodel-offer}. -An \execbrokerclass{} service MUST be able to provide both \yaml{} and \json{} serializations +An \execbrokerservice{} MUST be able to provide both \yaml{} and \json{} serializations of the \codeword{offer}. The serialization is determined by the \codeword{Accept} header in the HTTP request. \begin{itemize} @@ -2016,21 +2152,6 @@ \subsubsection{Offer POST} If the update is not successful, the content of the response ... TBD ... -\subsection{ExecutionWorker} -\label{execution-worker-spec} - -Accepting an \execoffer{} from an \execbrokerclass{} service creates a \workerjob{} on an associated \execworkerclass{}, -with status \codeword{PENDING} until the \workerjob{} is started automatically by the \execworkerclass{}. - -The \execoffer{} from the \execbrokerclass{} will contain the URL \workerjob{} in the \execworkerclass{}. - -\begin{itemize} - \item \execbrokerclass{} is responsible for creating and starting the \workerjob{}. - \item Client can use a HEAD request to check network access at anytime. - \item Client can use a GET request to check the status of the \workerjob{} at anytime. - \item Client can cancel the \workerjob{} at anytime. -\end{itemize} - \pagebreak \section{The \datamodel{}} @@ -2217,8 +2338,8 @@ \subsubsection{Zeppelin notebook} endpoint: !!str \end{lstlisting} -\subsubsection{OCI container} -\label{datamodel-oci-container} +\subsubsection{Docker container} +\label{datamodel-docker-container} The \datamodel{} extension for an \dockercontainer{} defines the URL to identify the \codeword{type}: @@ -2367,9 +2488,9 @@ \subsubsection{Generic compute resources} The \codeword{spec} for a generic computing resource may contain the following optional properties: \begin{itemize} \item The minimum number of cpu cores needed, set by the client in the \execbrokerclass{} \codeword{request}. - \item The maximum number of cpu cores available, set by the \execbrokerclass{} service in an \codeword{offer}. + \item The maximum number of cpu cores available, set by the \execbrokerservice{} in an \codeword{offer}. \item The minimum amount of memory needed (in GiB), set by the client in the \execbrokerclass{} \codeword{request}. - \item The maximum amount of memory available (in GiB), set by the \execbrokerclass{} service in an \codeword{offer}. + \item The maximum amount of memory available (in GiB), set by the \execbrokerservice{} in an \codeword{offer}. \end{itemize} \begin{lstlisting}[] @@ -2460,7 +2581,7 @@ \subsection{Response} \subsubsection{Positive response} \label{datamodel-positive-response} -A positive (yes, this platform can execute the task) \execbrokerclass{} service response contains \codeword{result} +A positive (yes, this platform can execute the task) \execbrokerservice{} response contains \codeword{result} set to \codeword{YES}, and a list of \execoffer{s}. \subsubsection{Offer} @@ -2468,7 +2589,7 @@ \subsubsection{Offer} Each \execoffer{} contains a unique identifier a status value and an expiry time for the \execoffer{} followed by an updated copy of the \codeword{request} with details -filled in by the \execbrokerclass{} service. +filled in by the \execbrokerservice{}. The \codeword{offer} \codeword{status} .... @@ -2497,7 +2618,7 @@ \subsubsection{Offer} \subsubsection{Negative response} \label{datamodel-negative-response} -A negative (no, this platform cannot execute the task) \execbrokerclass{} service response contains \codeword{result} +A negative (no, this platform cannot execute the task) \execbrokerservice{} response contains \codeword{result} set to \codeword{NO}, followed by an optional list of reasons explaining why the request was rejected. From d50f8f60ca0bd9d30b5e65093657a9662b780b7a Mon Sep 17 00:00:00 2001 From: Zarquan Date: Mon, 28 Oct 2024 06:32:23 +0000 Subject: [PATCH 5/8] Ignore the role diagram pdf --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index ca63303..c977e62 100644 --- a/.gitignore +++ b/.gitignore @@ -1,6 +1,7 @@ ivoatexmeta.tex gitmeta.tex role_diagram.svg +role_diagram.pdf ExecutionBroker.pdf draft-background.pdf *.aux From d7e3a1087a1cb19b584ab556c4448843a28c3dad Mon Sep 17 00:00:00 2001 From: Zarquan Date: Mon, 28 Oct 2024 06:35:19 +0000 Subject: [PATCH 6/8] Removed role_diagram.pdf --- role_diagram.pdf | Bin 43966 -> 0 bytes 1 file changed, 0 insertions(+), 0 deletions(-) delete mode 100644 role_diagram.pdf diff --git a/role_diagram.pdf b/role_diagram.pdf deleted file mode 100644 index 7092ae81114c3b3be836724c51bdea8141d3c782..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 43966 zcmbrlWpo?M((j#^84|N&W@ct)W=5Hr*)cOSGc&Ut$IQ&k%*@x>`|NYh^WJCOFR!H4 zGxhK4R&|Y3vu5-Yk*uIF6&*Db6j8%jtZ$t0Hsxz0{BA322S;v+xVxLQ~0Z8#V{y8-iFTpTHf~&8zi^IG8IN zQJ+I{^-$BglsR!~K+);Hp!ih`6`l*RL&nXnIc{ym^0vb+R**MU&Rog)(X0-2_9(OU zI9Ja?;l6-A;G|(02eJc9$Zyj(1}(OV{;bcJ)R#T`z9nU>{+RyobTMcu;h0Dq*N6O1KqpwoVr;B44Sg}YS+jX zTaG1Yrk-otZyL_{gEMP`AcXLS!S9E!BFV2+amZ(Klt>`KF+{lvjgOR)n?(iw`6xTz zeFbXngtQ?iA#!LkM?RoR=5ew@lTN-Cz*$4y%;_&%hnm__zOY&X#Ha8C>VG$GugSLW zFfSC&|0MWUcvEoADXkL$b%W>1DfuNl^~bv^c}`OTs~O#*I@-Hd_V++(57!>;ja7$y zP=W~ic@pOI$F*GUL1W^^)sSBc0&X?W+z9Rk-fUT z_Hq}9$@|S=%JUb1S@^{ULvQI*5-?716GFQE`Sy`)rP=u#4y0-~q#Y^%wW?l7#ba+y z5F6tpVHV4VkcqIeb;O|67T_Z^L8nKD9@p4T49>ObjV{|KIY)#RxT@Sds!05G46z?n z^Vwi4cL*@W8DZ3CV6V9NS(Fg#rzrN5j$}t2M8UIjVA0kAa`6?t2cTxV98!e09Rz_U za?q?|P_ZP(iOL-v5Ol zv+cVW=IP4rWysJ-8ftn?c7m6p{9Y;Rw~F%8#Ad8~g#8mKc`q4lxt~4Kcp16A>8zWv zlP1Xkg~&ZW!i2MYKpDENWUpGW6iZp?2hwyAh_$^3G-rDk%^=;MT&b(pg*Q}QGSC@neK^$0$3UTt9X6x{V98Y zYTcjhXSJiFr=|bn@W*8McUAnW4hp$Ah$uLG)<9fN&OgHEgGP_`)4-(>;KQZEr7_U^ zY(FdGXUE?yjj*+q!+&`jQPEP-;xhkJS^rF6L`C;0vHVYo{!?Q8pAy5T#P*N$Is519 zUs2M3s{hn(MOPaDE{&|7F#wkcm+r5qW%cX;pFz<5GnPgQU}&nxZ|#Ds@hN>qLraUx z$V7{)^_RE6XTYEC_PBq}2!EG<5j$%~oBvwG-|z94`cD=VaA_3n^sMY{{=Aog>p!v> zu7jN;;2%ri(@PNGWNHAA7vcXS{wqiF0DEgkI|G0{?w@?i|JTas{>1ms(*AW?qWjN6 z{O1_Ehd+_h3H*<9RfemxD$>Hj2ldD(rMOFDscH+68 zvcyrRaL)E%&vW20o-i^xW@y*azLOsi1O08g{XnGe&M_nsB6esDK%bI+x{xabbDspSoeHp_4wYwO5~(xqS`)|{S`Q9ky5ngX@ZBjvfCC~AkX4dlz@&E z-_6$0M)9@F`C}vVLq9!Yh#fP|@nc8R$#0P+55NF02ap(6Tz|cWyk>PxY*&4kj2sJ0 zZ6GV8ctxT>DnkMd6$nM?v+UEbhrU6(5xhy-_g2VJ1ZhYX&sCOhm?co>dnkOF8h0S= z2-Rzb%}N)SoFYjc*HaD{r$Wy&{_ zd&_sqd&xGEMWAF*cGjZlZt2kt{hq)d#2>}~tsfB3ojbc-+D8@kf-<2>bn*+%*to`i zNt8KL>o>JA^b*wlH}`~&?oI)eV1auRT#UfA&a)fk9+aZw z-Ag=Y8Rnp8bB;+J%h}aBG&__#Bsz3FH9f^VrQUeo)Zf63$MGtG7XnWDm+bCWZN?MQ zhR61ST4SjvtdUKbR23Xw!RuR|Sit zy=IB_ftGxHhUZit?TNo%7kjXB>a;Q#7`1r2JyJZ;x|Yq1(jClvDNH%`0a|e)&2S=tSj>pb{n#Z=_HYTAGU1Ag&!fHt|e2TAr7q{Ao0 z4e|znUl>M~mK&VQ&}vTtx9;suY_kf+<%>e~3YPfDf<{)-J>7gzC0dgC8p=2w$trL` zwW!@_xssZG_B=7VJXqW=*s1et^=s}zvX^s2rY?aiw;n=c(fM@L2G6TYj`tj|7#27|!(<=e?_HueHmj#WzQIG@uL} zZlE(}#1oSyqN%VB*M%n)FP@IQ$I3VTzVYFV1`mZN{e=vCJDBtxc8=2e#?D2X9A0mI zd#U&S1`Za_0KyK%yuN+muLNS*Har;nXf*QcdflC-=;jMFQ_jYJ(=kRYaSHdM+r z;Bd(zm7UP#!+UajZT8j{or7A<#$(zadkLCrWUq7?ua)Pzy7Hv4N+;q!e!KK4--Fy~ zdJ#KWfG(lH(6@WLs`czxB>P_igeau7TJFiLuk*PZj9M%bC zYb({^j6_gE5EvJFo^!_y=P59JH!V~;-l_fIh;37En*sQ|!_L)&Fa_1)Pv9kwUzk6t zf$zI#Y;;!PWzMxgcLps|Lr$Z*d_e`Tk5HTaNVo|s|EfMeZ%e;bp4QrO;( zi~GJE>zs7Dq}41UI7yjs_YHxs8_g~MZBVCp>Q=-TKJq+?+fkSYn}TcomoA#yjj^t` zSvt=kZf=>#i3W|5zs%$gPB+-Q5%1odZvu6lJ~8MqWc_<6h`R%SFukhQkRcPS49Hjq zp081_CzzP?;I7%P#8^D3G@TyKvaGSxZ_j!+g{blGv)KgF(!PpuK-^-(6x;qXVhw8rgB_8Z2zeE_S`yEC|Lrv<9O-jzVA4?eTwBdw!ZD z*_pT=i%?pC4?YK%p)}i{$Xr&?p_*sHsC+L5-pf_lXvg)c@fa+>3k$iHRvFjJv}AYm zx|0!)W;k@BR^@*bNQOwfcJF)-T=Eq zK!w!R8Ur228fhD_~D4uRmsN+E=e)sqVP%Qd?}Dp#)CGu{PdbjKr5Re+_&YSgr}-> zXC*j8RZa4EoBS4?NLbjQAIi57U_&ths4}5a3l!d<>fnmKRWrffZ>tiJ9J<#{9anzJ zp}?76k6%eSk>Ibe1IS{2Vg{n^Qh~fOa0Z~@Z25e7NFIR|sZvbLCpw+AmQYJ-l?}?O zmx)Y_|BjdNTpRoo>uY=4#`X{&J{p?g7g$yC9az$ZFDj5(nUF&E!4 zjU=~D34ES6pKrt}IB(y#upN*K%oKjHaY0GXtb+1k03pS4o9N-C@I_S<^`h}0(BvID z9<3xEcl&_&36-f(O6`6ownBr5Bg{=KS@cso&e1>hNnpgH$tR|njWhgPD zS=ekY8XpEVSwr*}OC+b98aU%E>1ZK6XAJ$T4VMr(J|AdUde0}E|2%Aq29N;;k#lg&9F0D2(2I*1u6$F&BE~dmadVwEBzpcLrPW1t~Fj-~g`uZW%*ag;tN+YW--{>|dwEpl(RQy>5fxo)chu6pai}OyU0R!U0&!s_8_qs!)$`I#m zh#6sKiSeq-ysuKXI7$rX-Zxr0Q2=9bV#q zwizoIr~u){aPR9wx0{}#owT4Z>HUTTM857Y{7J)ZGnz$Q*%(Q2BI}>n5Tlz6$wM;H zWi6&a%Usby5x*@_epuI$Q989#=@zX3#p;ZFuSA4%l@{Q76+`bhs0BvTjEacg4LgH0 zUC_nMhYbmbf?YyYQs}VO8S5?%rJZUXedbvxHGgjr-dy_1&JP|aq1O!=Foxct-p+rq z@pC!1ei4Ub8*vd8{I@J8(EcwEG`JpJYLLw>+8^KuT@joWcWpKt_e398wsUu7+jA#D z+;2JVSZ~Ce9B((yov;MY3jS?_3?L`O@4vsgprypwUrM40uWyMoBJO4|=n=Eo_s$Gq zo~5eYs){Z)8$aXSFLx$jsxit$-b zUA<`B9CNOc)*q)tQ)v@3BUuQ{>k2fFfT2zr_w2{JU8=5NI`{Tyg$^jagpxYbhLuud zMUab}t);H5SLkg@vO+e$Ht3bs1a9M|Ut-pg0ft(U z*3Fl1#k%*dM51t36xviMmzzp*<$^pw6dZ)CL5#od0fAD1w9$YJ-B(Yx2P9a8&?Tyq zd>wnzb4c)d`Q3N2=Bmkr7&)EIedbqD(?BzfdPzB?+!i<&yAqMpKWT*FNijI3%skvH zG#$NQ%`*b2#E)V74RVuMWr>#?r~{}ldg7#pNh_>j)|Ec3!Czs?M9LY*1! z4Zcpy?sq9bnt#jK48}sYsVAl)ndV;(on9OqRGraQZ>cJG$O^8MG@A0O2A6UKY}yn;}~JA zkXb~Ijw*XeMnRn<k)SF5+epBk(xFPQYHO;6~%7dYgPUF<|t= z^Zvky=gEir>lqhY%TQR~h3ZErIC&zYLMBvWaSIFaR-sq)y$AFko>QK;`$SJj#71+4 z5Ib!Z;)6`%)n=vr6Yh8{5ZJQ8Fl05i_pwIqOjlIS)a`l2>@fX)Tp#DScL2TfF9#3$ z-{Nu`J+)>i2nl9Jy-?@c$|cChniw`bz(NxmL>NseG_7Q}?A;@+fziz5*?rsNuFg{( zGp4VNJ34D)wpxwvvIS<(+0be5)agbiBDS!4WALQ7Xp=DSRQYXdmlo(mLS{iRrT-(K zE0^sw{x?2nvfw+0;cK7Pa{w>6!y_4=4ZMYbhoqe{Cek;N{9cM`+>|mhY8+=gX--1H zJaGjVk>Te4Qz@a=8ONYhN%$Vz&|+-3A3O+BN0Ixna$^KF2{zbbwS*k>0(-HRt=YW& zvcAf*ug2uvWj~bbN%Ul`Q!mk_K#;Zo(0QS!`R2_I7T z@;&HfYsY7Hd=0unA~n+yc!+HEChI4W7M_@QfCvn0_2_X@+D0|z6_|yX#h8_t1(>y% zW%ns=Vq1p5MmPa=hK=;7DVp}?bV)y*_V$^Xn zb`>7!s}k7MGSMU>H+vep9J}tj;1P_@FC<&kN3I2RlLDL110v9yGk$xWfW0*kE0B@3 zNboWm+S5L}0e`=(eok9NisBJH{QdPSU02ak3vr1Ip6@6-k*PtOW;GY_uUsb9YX$F&~>Q^7pQHdiZ6GX5$@xvyf{ zJ@UKV1t!U@y3u`Ou%X?y`n@(8mus|NB0B-B-5%sUyBmcZWLud>K%8ss^VM z^?~K(P365!7XF=8pshh89aPi-QM&q)P6GmEFh9P@1qy+5?i7(fRMIqij(a*lvBWts zd#G*544;;wWMxC9cN44eZY}701~f7Jgt?N48wRY7X{$Jw7#M@^)a!H_ z5y=6Xlwk5Cj)=F#iAgjE#HveN8UbiBOmDwcxzWBNUma`|as8CY=HO3G-L|@2xNK#J zd!Qy?%L`SF-6{)@5XCxlPH2>~>>L7Ue6y-G(|5_U*?FZGt@l936K4ye0ml=iQVPsQ zi^Wd7u58`4R-lAuiH=c?Muqzfk7SS7_TQ40%>APy78Og_bvs5E$Ztr}p*BM0_;N8N zs(GqKi{-_N^~ocnSYrZX3Zp8?#wn<&NvSL;-i+3HZt~KKw8(sYvI8de>QXP0U(Xuq z`W%2f{W}GCvR9I+M-_>KmIh{NVMo#T+3llTM>&nXZE0x-Y$1P`Sr<^snn$*E}p5pHkc#-!p_E>Uqr~I|>j?64rkq$_1D(EeDiN zu{nTq@zG3iBxnkiL=TJKC6q>))ekE%sWEBoRfWn5l_i$VmpGM)D=L)|l7Qo2{m7^^a}+Zf2#9k=RPKM^|6jg61L#)waTG zRITi}PR(3cy~9?)4)WgSi5Wc#bt%}pyucCj!n&R+Q`%tfHuYxVhIgLk4|?gi#I?9s zgf63HIp(_c^MYRoCc!T;Nn)-fv^Q>n8>iJ}zUOT0aWWv*AA#YLOE|cZfYNxX+at>% zLoX>o|40!FWS~lN(fso)=FE+iywsJX=B0RJmHz3luT+oJHIX@h&|x{??#>_Rd8zacJew)QoaevqDB*jO(dT zQ>ovTQ&T{0rJR`w0(yf&JVAqJCk=+U$ughWmg=eV6tjh7`7W@scBU7n3rT9{-p+@z$)~QJnPstLROlC38k5`505_`ef~vEK6iQ)w^H_U@X@~QZ@6bKBwDswc8;LarT~Kz3-71vkqtzi?6sIlYzN+r>YciGrH^a*!B?=X?*w(AJmYNcX7 z{3_r#bdfJIQsDt1jHZ<6TBwMJe zt>zOvr7N}7SQYxqo7Dw&7T&HTVw>*%Ad1CWOjZ|=qSCofAZr4BejLcXq!!SU!RB@KG&%o?=p1Z3KA5`4w4W51;yJJak+ zL71Q+3Glp0uL%wi0Cf@1Tg(ki$aQAt+JCKYb52b?Ka_)(xHgNOm3jmh>8J9+HtPLe%yHEbj{^X?GaAw@RfhT%TBrSZU)u*SFJ>7}qtkl+kjF zqoRDrWap_V#uM4d!j5)ce?WatqA8!6^aHS|&PO}Q+Tj&D zeFdj-iZ1;%Z33$x_S+;xr(1LghVsMC=yIIK!WrxbX)r_0nhS=Bi)h2-VQ!H|3Gjxe zVe>%Kw#!3wrhmJ85<@AAn9}+cKjyLPg{`l7Zd-7j?*Yh&&rb&pLn70d z6GytZTgQL@4+XV*fS}}Mu_1PL0!0)KA&fAo%6EPQrqW}v9#ZQ_7(GQdL^D^e9mEA- z%D#=3fT3bX)%MVzayuRD0@wA{jEew~iQ&~OOWB?npq{vm*gM>^gQV|pPBw1&9Y@?dH&2K$h|AA7vltH(QStj z)vHlryE%1nc47c??j*7j@s1@d`vcLb|G_~}UENSvdA${ei`lq#1b!FkhDS+-hk&nK zLSy)Yh%q-6iJO3efy-X&zEbhZWz(m{k$O140<-S=NeXk`dKp*yRkrq)n_y0dk;`i z7{o&$2a*#$Y0-Yl@TDR4avt)jk};mhrpty<;2VA~LS&epM^hluz#+kXhFIq)LCI>p zb|aJX2-sT5D8oqEhkHl)hWGr)x)Z@whWo;^dx7X+8tEgvbq<5W`5j6aJ~j|Cv8n_} z4#aalG%1Kk$78!1uW;&@V!u~%2Sv>Ql@NX*P|P$PdIib*f^S`A!)BY1+s7N?c_g%6 zBm2FP`y_OO7?qPgl#L(q)EQENF@r`NP$vE7PeBJrs>z z+<<&;2nSF}CDW^|WiH4ZAG#ri+c2!*Bli3Xs^yjiTIGk?mQf*q?i;eZhl6yHx6VS89sy@y}v?yl!oEY#!=kCpzyH& zW=XoQr18u<5RXhez}X%NQ>ijRVp5~PkWfy#iD&>-I_Lw%Es6y~1^C|tC!hMq22`xv zUk-lh^B*ikguIFl{?+JkmV4@bT&I>xA@@EUbP!5GbL5ro^}93aVf1~eMa?zkS>-nM zmFdac5w;HyB+Cdbt)hDC3!&gQ3y)A6gkN#?%Nsau9^1Zaaufc;G^Ys&uX>5H=uY=T ztBEcXclvn!qs=NWA+g&9YBt;!-9=8dTU5EZZVJjfO>9UobAssF{Yd*hvB)N@ zm){h|7C0bla!|zPTl5|C{I79hu4*i6m<0Yb^x0Q_UgUSe@<3)xfgxx_#C@7>psgBR zFGx1XA7^k1q^l8LkqnT_6RE!U_Jf4BL`O5&Ih+N!#G)+Rq?aqv!Ipgzw-RBs_nA{U z7Kf6et##=TrQ;yF8IYUo6cF=DBr<%P4*`7}XAweD1?f@qLV)}h;srAf$l`t0`yHKZQ$NvYEt-bkxPCeHXc*9uoE*bieN z@B||Npn+cSpx$^zr3B=>su`W(g0ralegJSjfbkiJ$c84_mp%+i<>IyIewW5ZWqZik zH{?b$JMKqA9+PT1t#wYA9~>Yb?N*2^O`dstNaDno{QCjYVRs-ahwCFzKF%2`PWo9f zAz{CoNLs1g*tHX?<+z#_?s_<0`<^wFzdt&{5dS-|oVWc8lrc_jd(MdlKbB$qE^LLo zTUqYc12c`O@YMZFqxa_;E@Sx6;Ca!V0(jDgT0xyef>RUn9z2DP*|97_^j`2n3hBW# zMv=HT4z_H}%jZD;XY_9H?M{D`V3PV~A=(6>Bj8Pb$0AgHR4AmrL>aTFjG@lM~kCm+o(NLd6f-?Msz220s40fqgzmbiY^j_&&;(nx4m9}UgtYaN>Xo`Zwmf#>;t(e z%}^(y&Xty8YnV}m!qoX(&b3??j^|Vpg1*GGfoMlIbvPW(Mc+hb?(_rnK~Ns`|8m6s z!D&8^0|Uc<4%mOQhkvsrf2bKjfW3j8sf~lR-5*rsFOWuB&+-#)5fBj&S5~KxHnr4u zv{(3qPN?{;Ee!wdB&=s?YT=4Y`gcF>9|y9(J1z8#Ke3j7JM#aHz);b#vf@%P(6Zpt z(K54rHZ1?3Fk%jR7N!P#R>l?pT-v{e@!1>vL1|d%+5R&Bg!sp%qG$OVwGq&>5e1kU zn|#tXpVR#1qu>CrRK{ifk0JVx@n?~rpGE+~r}O`Zd-($h{e!*yCnJB)_dnQ{za)M= zd%$0u(f>0wrgruY0w#KPxD51vjgr*+m;L9v^G}BVlXP6V{{TQjR-eeHsg*GjE-~h)9!hvh>}na@28tu?M=G~l$n>LD5=0fwU# ziC&z#sJ~pZ>+HnG4}9h2`4Qg6!6CJolbDEF5+_xc!|-yU$0Dh4@!hk}X}b22*0bJ{ zhhDXY(#}> zLV7^j6AwDfHHZ+Q@R! zJFbsfjc7rcHi%NaXThW~fvrbVJ7$6T;_unlY5bu}7fS%8mg6-DDxJ?r+#EeUm~cW7r+Q7hmmXDMwcZYc05e3W`sG20>q zHD{(%vHZmBj@Ti#onghX!b2Ue*qpabYd+J&QyEpVZCKSiXI<#k+Oo2bY9{&QmS7j= z#&Xk?nyl8iFZLJ53Re8B6ke*+Zpq4u%6m>r<6#lY614056B!DOX^uRa5=hebx?UPX z``&azja-GzG<~V*Eslm=+Yf5rjK}AcxM>iw&d7($C}r`bU%=jHn{`1t)Ex55NTc=cucwnrI-gtQ`0K~EcB8@%3Q!y+5Nm6xn} zXtKqxpUPDilZ&#oln@{xy~g3dI+FDD2fmFh)K3^UEvg(|)(e{*b_NgMEVqqCbotaS zZP?bq4Y@-?l(~*ZWSr9;o6vfo5H%7hpw5V^_?sMzLpy6UmuDLWSUKt`Z#}zR6R2i5 z+oM2m-kzFx%`_gmx6#{%&^?mWz~SkDSg49($@p7h}hEoh46b)yRNX?}~a z{E=~$REdpDf_efZBf{fa89-+36X28Wa=2qL`6lk?N!jQTK5!{OOg#F4DAGJJfb`BE z_MP{apWZpo@-fb9K80g8_0(ZVZq-l5o^Xe=^$Uzf>!GE373Z)a3GFtv1e`A2DG=M> zm-v;!r?cr-5*Q33&2cN%mZ#2ZtRAv_2$#I2%bw-bd~!gZmsr7O2>oW$}rQQRY+{du?XW#3Yxjuno^aUuy&21)-K)Q&&<2L z{?z&B^3zeT84`U#$GvY%9OgKtTLI85>-J-M8P%g3kf>-D(E1v z7{$@;WaZhmv|vBrc=ZAU{b2b?{POR##4y@&o*{So#LgwKUd(H*C!_DJ9pcWppgntf z=+U5e@NNYLX)2w2U%q!%Ui9s#6k=l}14|pG0oQdya%{0Jf^+QnEV}<{2BuPQz-GBU zn`j_4F2@z|;p_If!SKbD!LnkVLtCYl(KJ+RWMQMmBO~MH9+GXH&|M_SWdoAD`z9Ny zi*-b!x&Wi+IL{rXS|5?_gcn(Na+4_x#(`oAN13jbFj{16lyAImwU~mZovQM~zFRD% z)n3AhyUX*Waa(J9iUT|oQN`oE+A?uF?1!{e;D^9QA;8Km?3ZzEX6l-ZX>R)4hM(65Q!*F^}T?zRj8< zEMb*4kMk#hg-KTI`4F#XNt2ECVMfkEK9df3~T6$fsauTY zgrmY*XOzi0clYa>qN%tbXriKd$nR|ip7zLV`Kg$A%DmP{rMEF!NZsO$!D};ik_$Z7 z5fV1F-Y>cFuByLoQI;R-5&5hc_E~8N>6U< z5nW~VRC~xu(_x3yg8V|Sa!f8xff+9{j=J%r_QILtc;%{9w(#?N{wI{8eyE8 zxXo)y7C>eCIPSo)PGM=Od!JNins0kJfRS$qd_3(2At04eU(z-!cK3pIr|4!j4)K)= zkQ2TH!zG;sZj^UTCjiqT2s8{L)0SHu*qwvZwl!F3g*2ax1BBr#$!nXYR)liljXzd~ z;Do_7Z+lB&#{LGyA)5}S(kuWvM)X)qI`BoL4CIL>>94fG!$lu3!#<1tgoD#r69B#y}S8Kb9jSSx#L7GQA(V@H8gW!Qpo(h zrLW_}jKw$wN;Rt7jS7p`7 zZ-3Jp2idXgA9)UQq~7yno^vHGM!?{v^bAlA(I6DnM2D9!h&cHiJl7aUj|FHIFnv^qV8nD0BIoT|`{cN(d9E|^#Nv0-x` z(1QR#O(3qZr(sENezg~X*os_J0*zk}DI=azC~#MlX4bP`WK>XCbRsYRTV>!Xr?P=@ z*mW4cdZ1<2nh_fqHs7rx@5iBb1<#7u=h~>wb)|BmnoQ;|JwB z1?~@aeh{9m;B4MmK&-e69=1==4k+sv;b`!KSW7}zxi28mc~gr?AkLzS_1yX7t({hJ zKUY=Rro$PLn`T7q%!FvF^NeJ(P#K-vACA((kab64QIO}Dg9qR)xLgIGfl5nDt0F4l zkmaKGod(UbY~cA6!x+;j3H!qUhT$#hQyyH#}a!KdP$j)`)|Si^miF zsu0XVsF62>GM&#StZ+U+I;6|kGHrrDu{7x&W0)T^ZG&+4lkP}2Ki_X_Kj#T%V8T#N znMuQW>*dh1m?%i7(9oGIMYxa=BDbGD7A}hM=+fq9(j#!Dos$Q9WplzP0*7(32)Udm z0WXGOSZ_}uAfCo@T9*d`Bh;_@g7lK8E`$7t=uNSXs05GE3`w%eA{3M{e64%7x~?Av zYxfRuc@zRdOaYU!_cljYqf#J|~7M~1Yi>{2!L^VfPskq6?&Z}Nk)rB0)WHF_!65LME% zp{uMU`3yXO!9!U6%ad%{!DjK0+i=A;e5tbvsp*>$1_baT^QdXpi^#s2v2Y-5?&39j zgdb9xFfwEZnpUl8JBeDn zckYk#)Qf_{Ww(bVG-<8AG=J+ak9n46I*LMgER|Tq{E>}ye8Mxh^s;TQh3IRy{Ge*U zi3g#>1}RqY!8RX+Ti@uAQ51oR5LQqH4$ELi+Jq^HgNFF_ndY zJ~dLE*($^GGc(-18whCh5?wo}8r#@s9TdydjNPL-1oz@wtcw$4DT;6L-eBKukN_c% z!$(QJd#KGLA_2V<9hl)yBLd%bW-T8r9~@v)`{7w_#v;kzo~D~`*nuQgG6>-=Ec~0n zmglK_7EfBqmb^`S(cT((XCgXV>0vr*#Poyp-jVfLf`?i|5wvk8-{vveM#x5Y=frO6 z97mE9PXiUA8nX49!S!0}{g5%Ou4MhNNp%_SXEswFr%J4dM;;jlfEyYE77SScJVWb)RJ)?* zgwAmj(G&d>Ej(UG{m?&*Ue(+a)7V6+3BVhyPTd5eG%9vD%BxndQOdywAm$mo0jzc} zqv?CQjh4Px5}Dmo{go52b|O9ukXEM`IBxY0;TBEjAVG@V&r)nHGy}L#_0|5Hstc>c zf@h=#Yn(9+*gvGGw+>e(J*g%1OYR|O8`ZsumPnvBXy>cIs|`zbF)d;T%t`@ButQ~B89N!M@!wOSDxu9%Cdi%V$q8Dn7lo{F( zir>dAl(5~hU9hc_I4ECzYT5FeevN*icKJFRkhjMm@qI31yM#!Qu_5C*#4ImUo~C$V z7PA%eOw9NbLBiJJ-N?#>`l6%=4mxIVHQ27O5yx@ABJ1mOZlce zwt~Jwyc|v~qnh2)alEvzbOuTlT6t`_8t9}q(ukyPX6ea-vn_FL&J}H zmmt|!(kXwZ1S`u{CS2TKs;Q)1x>U+LC$E%DE}oFBUb37$taPTFP+Y1Mq)4J@tdyig zRf?nZ`M)xys?VE;-lgvq%B5{(4Q3p7U+!@4WbasxOU!U!wS@)uc+B|7O5K-%oXHEB;;WB?+PzA2n?*(D%=qvWc=yDuFB|B^*&N8OV z?{k?m;_3-*gw7MTS1hM*EKwg_Cyc&(sqQWzNoGB3)AF$0npN zDr1Nh@2f=y-3n{yK>=MkndH7-DCiGl6PejVDWWYoH_<&+;HnSJ>$$Jc7yu;_EedMVh zjX=oxK>V%T83r}X@Ay1C`)t45wO9Sd1MWlNAuwxG94!*ur(3uhSz0)uw0eKZI_B077A8KF%y@ZxcQz=xvD3jjE{+Nv zCbxS?QL-a{L)7ox6hrV)^22^KeRX#b@JerJq&5&sed#TB2fJldf{onwTu1AO6joTX za0Ffz-D{>B*Wy!%TZ+Y^Yi*KEE7ao#^QckBOR)1~^A+d{6bc=zZZN=eS5zwx%RWE` z0`6;|9uV8pWK63xhd;tn@Jn(!?MDfR>JK_!C}{~X=pB`c@wG(qNMriYP-c{yganQU z1fGvMEKSREIC0C~uS9wIKJ`O04ha}U8O*7i=tBanl6drbGAv)azf_B8q0=fduz4k2 zvRaIygL~x~$9HLSqO#TLgYf7H>)RABQr2Hri0A|L3L4E!Cz!7>Xwd^t_sqm=r-i&{ zQRCL(TeD#>YCR+&d2hRR;^=J`{-nIervgP_c`pj0-A87M@UlF|L)#S?>Jc97`l|LJ zcYMk3r^8DnFx$HKA%8L>q#1T0bDyo}59?tM>kAYLvd+Dm$%`QCcQPGD+v_wI&9=jb6N(!05s=6>`x}F(lu{&XMX|mDME=-S+7B?X<^+f|gkXp-~EmdWi`|`o9Go zU$Gd@?C2){T!UZ<^Sz>tglI+hKfJwTlw{4Ku3NTUUAAr8>ay8o+qP|VSzWemTV1wo z-|F@4Z?ARk-uIld?z!Xs$Q&~xXGCVsm}8Ea5zq5REvLOsroKdNK6)cj4jbqy5=K_! z(6-Jr{8~4TyB|(-_?qs#zc92ux_DgzZDEzl5I9(%*t08LRdcI+rT{!I`qjyP5>t$~ zcnRcMKGSMjaZR+6Ek&Zz<9CRq*wLE9^cGG`BO?1&`eB$6@B$PJZbOh-QVk60$$IhKF{=(`>B(qP6v zXY^oUzhTJP@r-77A3L9lQuLK$Jj|E~ny9wkK9v!uMGxscX4hc5f6gn-cxu(_usqWS zC>D~WAEFQL;09jpqPQ>RUVdo??&thd7U1U5Su-2=ygX~(8|}PUPO8Ui@0t&1wUe#G_a_wS;DPhYCAz;Z3UQk95n?pF%+AB z8D>B9U#DBJo#gXGIK~`}58ip%C5`U(@V*vZsWA+CDDm&-F^vt#37kEvB7CkxNTI-G zBqXF$cVm^NfK7c#wFyk2dm2~?*gdaDHvTZ>aPKcyHSE65czCmTOd8xzqMwf`@W~q` zONg~N%cQWZ`1oLJNg`G<#)^$ew4Q`o>#F1+hI@7Q>hOzBC#={Lv@3=0)|wxcR+_98 z6HA4Mmm0d=swgQ+Ds-FZt?Z2UD(TJkzh6d{vqBHtR?_$OQcFOzwOof!3g04)z-6<` z%IH7yzvh!vTcn<^mW6xfB!q8wc7+ZthP3az*6fZgS4W;Slv1Rdfw^37`myLw}jd|*YfP`n1xFF*%XvbXFQ zj>^E=Z{+tJ&CvyU?we$y26|NTgIHrG(x4J|2O+qNsJj)zBiyzxH*mSuKZpH9-ju%5 z+(NAP7^uO(X*HvZ)$+WKPwbrY2g|HZH#c!__^iwF)F&H9l(L#yZ_wno(WLEqvpQMe zCB%(_(T?z^E*jSMyr3mq}6mfukegZ z;5+f1hZ`Wuh*o4S=w*TWbW>9k213v++UkG~U|y89!?bvVIHVOku)YLsb$qS#Uw^Cg zk2-!8Mlt}F2=yYSSG~aei8mA?S`au8ZlPjaG(=an^vJEMt>c+WM=>ScPM*i)r0)QH zx83>i5+Ebv1U?ly40dybf+mRY*6*8DgWCB<@qzsf&P#NMH=zjsS(hm)N{e13!d^k0 z_|Y)Vj5aGmIifgnQX%?xpVla^hW(krdMOWjGqPG-n`N@2YyHY&nPendc zkf%N^TY?O}uRr(l5YgHLEO*1xmZ;6>K$(m=lPMZ9Qc7T^!>1f1jY2}>>yC^9xbdw;bw~idNM`8EP@cHmp zvQI^xylP^h)FoW{JU2%xrdvMv=(-L?9@xi+M8P%dp$!5^%Pv0l9-N^I)-XZTB<{<0 z<5Tpusg2KQh@W6+Ul4-agaPfVVo6^C=K$|d_db!k#6>UlXo($7 z8NiaXO1ecqsX_G~zRIrT>rU1@X!}K1}Zn0!>go!s|p5 zF)1P8ET4B#t)D3AyHYR_^b%2&uF}q^9xp3R)x4PSp2l0!@L=e-LgJYtxS9HQVI3803c_PVmmePye4qqwN9`4{It1wmR=q>) zHO8eJTL|zhstjRTSIkXjdq>WllX%M5H?~d>##>BWjKg-<2(Y|c^v+jK zSYg;D1aCc{uA6wLubV!_wiJf;XyhxxjK_|zjEMDzk%!`^HJ~fbY^v}A6JRL}mi`%C z-_qK%=Gw*|TN7Pv8nOG9_m3E9=IO*VV@j#!cucZE6cX*@o-&?Uw%W#-P>O;knUb@r z1y$dDh~E+2dKrJmboFVBI4w zi^b~X=){WVh(g^!?NP*ZbJJXxZUNXN^vdr{UUJPYVa&PT^bLFTM6ts^HX$GQR=&Lh zS&J8b{x|*hr!W447Gq*!`%hTx5Bu?VxBZi}`6mYRpJdEm?2&}9l;C&ee`Svpj7*#? z_5aBz{oD8v;}`uy!^rW6JYr(}!ws?gM-tEfA%4X2kFWl}g&+Nu^!Cp{p?^J>|9MdU zYVUuBAN?i^Zo-&>6T;GO=e^LN>wSAXaumaiIr2&KPj{3-oY@?ROGzp|tL*?o+E zcK!RL{*P-BfpK>{RSn4_jOGECOQk$C^lpReY}Q5H z`4}fT4_`qzF)wsoO(3cRfhFD)SCPeSGIoV~$_4GVQ*Pt&{rBs8CtvC&Jy)B<0q;`F z#cAs$5gIH5df0%$iaTAGmp(y;_L~rCC6v?ky*+AJNT@I&!;9wR9HDn}s>y`wWF{V~ z;S(Ql$Y;LKQ*8gX5UGBoxM5fCpyS){qT_hAkl2ixBgTi;Ad^ZS2o;lldr}JoazsrF zLz;*pnM~Tv$fC^ZU^E_|z8SxsQgup*$x*18Vv#If4V`&^c6OxN0ou;~HlG0uC#1_Q zEX19|E&Th^`lrpN&u2K1^dqwSY3rN zWu0W5&_TomAKRvZ_h=?wFPu6TvD@(1?>%E*49nOZyHn^GTdOtq=`-8;}b zL^z!{wWAtVVdyVR_dIeVG*+0LlpK{@k&I4#sqSs@9IPe`f(+Sj)$PxqM`#Pevc=2} z-dSJU^xIDGPB zzGN#%9S_Z>&2lK{n#D3}xqo*Tc9(WnaMyNMq+E_{T+!P#Mm9w@M>fiDEUh-HJFaQ5 zx?MXU%Wz@sS`D|Nu_RmeHs@M(ZaFvCm$=V4#3GdXeGS(DL2~0OJd%Jscdk%Z2d(k)R_NpyS+>({? z7W*7GNN$C1CGA4q>L2_cf*+zE8E^1GX@YsNlqgV=41x?Q3=#~wHKH{-HR{{tSBfB1 ziBMU2-{ni$OG;F+-`Cx5VcFjhAX!ne^XmEst|^5LD_4+6-Q^!>L8eWI~`jA zCUI&hw)du3G23)tyQkhb=cfr*+j`(Qtu?)kCJ~4SvDs9T z(?FQ&x@L`5Ofi*|EO`wU6SJwcdYsB`Ri&b^ESWOT1LUw0xN1aQD{rtMeS#vCXm=RJ zB{O$F??sk9tE?>km@;FBmAveY)3q!9i38wC0)uZ61aOA#$#6{sXQc-Kc8VJ(2m{B~ zhmLXdNZM~ZjCJb=#2cX0RE+z5Aw+j$v7=Kf9_y&;v5;)yZoN7^qt0+wOu#>hc!XzS zA^<5=YPa*=I4LBipnkTi!~FdQZ0YxgZRc^RAvJD0#ULG}T5XR}?ko3ieWBe_eA-_U zPVEW)bviw6)3q5iob!q6(w}P`iU|AmdN4g$SGBw_Te}$5{_0eYKW>El$sJd2((bIY zN0W(V6Dmd~bL|_R)7;K}Y!IfU>A&qk6aIX88}CGqp=Os0z37;WO8%j7#J9cKl#68z=qrD;1%lK4vQ0+XG`qC zQyqF~%7o3-hB2P5+sp@#S0UGESsWKeDtyUnhR1<&H3wiPC!WTWGy)A01|BVsEKXJ4 zqBHx0XszuwE&jn2T#%6`2`eaiBNiF?Gvu;ZbM*{60aBJsPYfni4IBH~;1o6UWIura z{>?tq2k2O)CCFI&_}eFd)X?BNI@0o8JfjmP4}=_u-&yN;@2p%J9nlFP)B};0QLkN~ zO@Su=^e2P4|2^B-)|0Moo@-5m^wdB^OoY025J6r1g$?|=(&(|$Z((whSt9v*Z1A^X z&_I)$pBU!4_t9WJi|?*-PeR`+IB&Sv59q-MJaNIcQm4V+SOYLdb#9{H58fC*gl0vS zE7lMXF)^+2zF{I@NrnWWVHN)crpkc=N2ogF={J3MWRu`9M=et+P(C*L{vs5?VxpKT&%-xO2WN;< zM%_0-n*)f3^J<*|h`H-&EoLP5Cc4ZV1ToJd-aOcHkFbO`<-4HL|Amiv%IU+)<^3kA z-%OtvUxl)FNZP(KH45*Bi2nl;c#z(-AsoEdAM*z)?mOmSOkb2qWJHkZlNw#ydQNqc}`N9mq(%3FmVO@yub4<&;cx@e%5= z`I8E#EDvTu11APA-@SAsU{y7Bj7xZWegMzv14I-4yzk>OEwX$fDxf!*7uIj2O4-Xg z*q4E1jXC5$MSqN0WAFRfIrF5*ci;c``+#(DYyan^bTJ)%TMYO))wVsMQyf8}FBjC# zfVHKdxUD2Whr_2|BL?kCtctow7B#!JU3>-a=Jc!%4MlnBWM#gezFrF;3Q|zogGLsw z(BT-sQ>GRvH{U=xinW2LR6F`cKdFw1?;f>v$;5D|zAqL7w8+uG(iQHvGi3}pG$-Q) z7Dc*;bq%Lj8i>2Gn?jR3&!)81&cWB+Fw-k|GyzT1_X#I)M?w|2<%8{A6f%jZDEXo3 zHmx!ZLUv^lb7dCgpX!NUm0Kh}zCZ3S9NwKzu1oE(gqg`>+Y+~4frK1=Z`vbn(pRnf zEt|+!`eufKz=!(+Gj-`-ywwoYdkG#UY?3S0>wQ(0we~&mS48kP4A205#(MR~+fYTV zpa{heSx##lNU_hCTzXMkBvvT6Ufdq%wI2|I&3BAWBA}o{0!-|HvhBUePW5}41%*P} z*KYlL78@zvMLEcC0J6c*1g?y70Ck>4)*9ojmQL3{Ysg;e^ov#lVcFm6A6c;BOcZTf z^Z-eE83B>|S52smmS{KVQloMgzb#^~3?zf6^s@ws2Uf6eNg#{!;xdS;o%N8s7QDVF z`0w|DPT#B!Rk%#mkX7M+Jfo(^{oHr5^w#^H46lJR!H(j!y5f`4_FN8K;mCDTx!JL9 zWtouiX$8EAxWVMB-tON8~dX$Z;x<902Rq9tAvwM_m-pI@z>UUMC zs--Mr8{C#@`z&5N;?XjHDrb_0V@z|F!iYIiVudzB^x!jnRX5AuUBS)|Gv?(|@6};cV?3`aw1q5LFFHE8kGj$G9;^wHDrsvm3LY zY%}$ip(hfb^?=MW`VkJg5gx`a#<{r3eLkCf8tn+GWe3-i5Vr|TDndnUNFFQNh8sO- z)?LP(2KKzG1Dvm9Hz)A2n_mao>N2{Z%o0p0c>KI2q zT8~N^nHywmBj}QoGXf5K_hc+WGW?YIW;iuySy2n4=aNXS5wtDE-fjEkyU!@WV2=$J z^z!`%)*ALv5^eEM$HVMjXFKM!^K1m9l$-#(1O*e2QFU}?i8jKesu@>!sJfp(uXrM zS{fu|3?H!}#Jj`FhAgc{kqc^QC-GoXBPKv|O6Z@AmAP=tt`Mpleu}t2H$vFI4;T+^ z`8`5V@uVk+@7#a26y>(~F6t(loJON)yNJEJA2Jlex~;=Ug+ z$aID-CbyuQE5{aeNw?=XDxDNBIU>GDOt_g((^J1igN%)Znb=>Nkia6r4B<5_2OtY_ zfP|59BtW)Bsg;$tKx!}XJ5@wY!OgEVzH@!uwJTsZ1V)_U3Tea33~uWJ7UYrv#e3qD zFm_P4`h!^T2>{*ut6^8g(7dKC?z>pF!mK2b^e#~fcM^C_%1+aPovc_)p zIB`C5oSoJgvuMW1yuKccB+c^MdK6$|YY}f(z(*d#c#>4jS26%OgW(TxfOq$&35Wts z%x1We6zjkqhH<8L9L~>olKvpLg84wiE7-gvWzudaWq&@f(YyF@YHd+8SdB75FK$6E zp2V-I`cf4)CeZqf>P*XusJn>+eql>HBN4MCf|RlGky)TKOt|52MrB99l%xX1(h_lH zrTTCg8EI+NnK*gSLP}Z|IvjZ&Hvu;@-i)M_;<4f$;Jk2tDsL`8%K5z>Uol+++Iy2e z>)9$?To2oyTy`CZ;V-2^6yQ7XHFB4r3v^nD%E;PCnKqw?WnZ^czPTt+G>}j05@0J? zWz{%wy&AdDQZY*ao)mzN6cT9?j)swxlwneGqPOBf$-#Ky1`8Y<=~}J3SKCqpTF3Y0 zZMF=c^`)3*rHOT2ryK07Gz0=TVr}3m%mudJcof8OLZTZ|s#|;g#|crKeH5017B4OJ zGq#{iAnN3;?fvSuq>o_up0JUCHpW=2f2+N4z$RKsdJDJ%uG=4dhps1WwoFoB6JN6h z(kDFC4#e21kz&xx;oOC7gRiSlE+iL4`d-f`iYEj9JP<7AFuGij3KByF^AzUG41Ac= zha;K_%!eP?as;50MW}ZPAaj%4-2EZ&W0M4NQ0f-5^a}DtJME2H3v+~;gtiA5Lk<6F z({C?}2F&|iNDjDLl^(+S3?PaZvZGJVD>N`tBpaikc^0?FZ68ug(%RjdAju?8>jA@2 z+&B}2PP$?~rx3`SKo1=#^+_E7IGNtS^;s^I;Cv|t7}`k1zKBsVPeyW$lyF@I(z~<;{2NcM2$opbil1sQw|W4a5Mv-mu6Sm>Bg)Sv!hjaZf+0rlNr%u^|!C zf)|Te!A@qK->0cO*};$M?F%n`gg;T&-c<=GY9Aa1b9srmeB5X)p zFd>Xom}ZCfqfha%+_F_lMx?V_us%mo9PyeSfoZ)k1)03=1TaB;tagZ5+Tx}JW9{;` zG#=z{U?$fNTW7UHgYG@339=-eLi)m8R(JI7cUu4!1pIc4S?P)CzYY*gii;J)(vE{M z8W+H|tW9QT8RxGv6C+de@WK{$wJ%{d4SX-SzhP;O=B7zY7If=P6PfM5=9pkSDR#qRT+z6!Jr#8+-*^n;>-3Lpm?ODoToC!Xxx<1%=H z+rSSZ#sN;SHRIL!;1-D(5i>k=ta;b~sUQ$`qT1lBpNSHDQIOpKQ z7xD8oh8gG9;~T9Av?(qAQjchaX2Z)4X=TNmtEO8>0SF4BfoVjZ?58^PE<$Bgf(s%* z&pq|HaIPC3J^kL}@=u@s9e#!B>DYDtK7yFVNa_Q|9}P7*wK+nym@dkI0Z)(`M7BV6 zqep-bc7WDu2$HnC0=X9-1RTmm&;SZBmoq=$xWP9sBah#6@WQNhtFwH|UNENnENKk^ zd<^l!h3lC#w?L{UGLf!E5Qw^ar84k(2>3n$MAp5@GARM)Z{SB7dMVB*KVbRr)a@=G zLTrIPcNO&{ElEr{@>jGXZBTEjBSKq)IJuocXC)D62-q_P`eSu-Xmff>=TPsXd}*mv zNPN+Ov*h~m6a7|UYg0CXI)6n2m~P3;D2FAPJsgV+e2X2AWaBjOf8BBiTf(F$5-p?P z(pCK~PH^+1&W|&tl6-=7QeN z5Zf5$9qtnd7mSaAk@tnf-GtjwiJOR}8_P>CTS766SL17>7Wo?jyTLW(XPA^fQKR*h z?e%Jxb!k=E(gD3#NR+&VYN&9&2$is=u!V@G2$nFGh{23lo*f~0oD#D_y?VVivm&!{ zy+XZky-vLrvjnqby==X7y#}-P9(g)RI$=8LZ;EvC7d7VDc|Q!oT7F$AWTWs=k%m01 zyo@}LyyCoU;jYk)Fq#3zYj6XPYsqWUYn*GUYnPqN0hx*|Sni zjSQef>mLQsf;?#pfl32fdW-jCmp7p8cXXhW3z2V5u(^{ndvA}>=`NoN({GB#S==b- zf%OLAtQ?4D1H>{&RH$c6dGeJ(2tJ5;?3s~B!XjJ#6o`T7ixikN0S#;HYHNy;s-( zQq8E{v|0`svEd=_aBE1hwt=}H*{L_fG^0pVz^RHhL5J|^XDZUfP1bYL683o z(^*s4ZgV|gun9Yca*P61vJ%u5M%6?{OYF+fY2YxO1CwJ+ngt)H zvl*yij z3c?JxhfRiHaxfd-$^>|lovo@g(Q`Pb3|Qcs{f7`t`ll-@qbIWvb}#ZK`ls3#r^l)EXvuL{Y`WSZLFW+C&bgqT z9}T*n6>`_zSGLu1cDb(?J!H}_Bx>hq<3&@90uom~ocy%HCC_nwUW$<3hunhnJG1l! z4Y2`1Ems_nK7vc$y=^kVNkXEma1Jg-KMldgozq$}MAE!|{IsrQnqsM9>iY4q zoQao*sldGlz>1s^VAb2GKgAI06S=f4^CaIs=U{g{i#ie3HBST$NbE-~9|J<}=m2;ePuyx}l z@*eawyI^rveYRp2M}h2^w^SvarXoovQ77R8|Gh107_6?QHg45Zk7?34eu6rKg>{lO zj1}Bc!-~@qclmANKunWrRlhOc>Tsp9&Z+jk9=3X-dZLD=nx^i&n#baG`N3ngYxTpL ztO?76t+bZBhcFqu(>RraCO zvdFT*GQzTIToH{NI%!}O$>_Nj>TBjhXOa3eIV(Xsc|DdVsUy*gZg>B{z9lm*2J;Mg zGM+A$E}<%(Yq)7xd%wp%iCgF=NmVS{sNCq(sK>tC{?z{a4c?wl3Kr_u>KVO~`o(&v zyQqq&4k%-&7N}ULtEiJuEr3x?KZUFSW0|hv zHJ6Y5YiHp&$;gDX`*VHn6e(=6cuwe1*zRHP;Jp0eM4Vy8!Od1wlRmmj zq!p@Ks9DabV}NerqJ?OZgM^lV&oi;KYz;J}R84^H-}JJAQx#^E3RF#ngsGYL z4_U=`el-}FVSO-F-0lgtx$IT#Gu}P|+fPmf=(2r1!k0?|@38VQQX~vfuvVc>DlVJ+ zr9a~(K8p`KPW41Y1i>!|>K?`NG+dM6)4jN}h zxmc5B;oi6o7c1!xYC9VIW?OLJ5N zdDM(4i$DsY87R{$tRKHc@9Zr!b$B+a6I0ch+YJad#+w=^&!1&)rD>FAa^T5Za!=p#Ojnk_hR@`gM`(@3zOBF?da~FTzudneZ z^I77{P(wp2$|GG2xKMC|klv;3ad9njmWRvT*P82Q+qGr9h)Lc-tYn0)t?H~xF;)8R zUC(`TAeeekra4;y>$i-Y-Ou2ta=n~LA4sMsx|&T}Ut$pM74Um6#*c_>)%GwqA|F@> zJ-V=Vb+IhU`ouA*>7hXd`Qz=eq^zl5-Z7c}1 zkJiLS?R$Ua5T+BbOyEiFj2oki?j2C-$|KF0`YOsaIKXadtm8}%OJaEYVUumJD$57q zD)Y!!c7Cy+vo}EWgqbO-EzXX!Hre?yTIclQc@}v)9&A_d8@0-1LCfZKqoSXRU$}@; zRc5D;w?>Q%Yi4wuge2ovRFdjXr9*H^+W3Z){tGcL9mGF1bcF8Z}up z=79?8ob{Adgk400PANj4pBQOqh_Y7J%TAn$QG9Eeo`IcnigPmq4fxc`T8YxV=-@Wa zSzEL={yc#R(6z~%)*{X z>fPSBKgjltK#Q=E+$^;kSlMAZDXF7tjp1I~Q>Y6-7hh@Qrkn-WU}%MPv43ms;h7{N zj=jBJ#cPUB{-iz%LcbJ1irzH8c3Q5wgSy|<@>{EAY6q2D$TA5Hoq$3RQ<(3>R*6-HlIx?(_wOnsaJCSN*(0vnP z$DyeS?TwxU~n$fLw zIukey=%4W_hvyvk?UEVAblV^CvK=@Z#fs!4(j)Uq>(C)-#F-%bwm)Z}0*srhV|1ZD zeTx|s6%7MsW(ytzAn5Vj;$&Q8qRD0riFiNjLPY%*HM=|R!e$c?(WMI$emx>-%7Lcz}vi0`f_A&IUn$fRN%?U!0( zxLqaR8!y`R`k)REnRKt2-Ea@|rx8DIT)P{wWi!&_(xu(Vy#(LX&@VgUEKEz80GB$Q zNC6t|moKCkEbV2i+;QIboMk-7me!R$3eiS&Z!Lmhd7YMsyu$(U;n~&eV!Eg#d@qa4#--* zvRU0GuI6ZtK|k-Hj@E)U8(9J4D*iMOJk-qad{PgTDC_D(Ow^A##C5$}Q~D&c>u2Qv zV2Pq79`baoX*~1u3tBj#T27Duy~f1Rm27BOk0S9R7q{RNVq?bpoJn?LQOx$`{#q1Dj#L`84yY!P1Om8>;ee2SOc-HSb(ESOW&lH7BW{f*J_or}{Zs;HNU5Hp3P zg@I_`s|MT5ci!t{z_@s#es9&=Ol;)L4l1eSPfeeeYFWt|BeucG#mo?se$Q@ddwhSm zdnijzrFz2AmJ;`OzyrYc2^hH}0lC)H!mq~hOBitEk+ zg&V^+z<}1`Ozy7j>3=SIM{g+N#4jHxth++RjmD)io3<+9@(wbJe1{HdLxovBJ(N?Ar{UZ81_=lQ-9(3h0SNJ!|#1$#9GJiF7*o>JmPD?b^p#yspc zNTBv)0vCpLbSzsaw$oD2En`iPlusjWgp@d6>qGMPxW9pwMkWhzqa>AT!I6TOj=Ceq zLFFRMS<{2nv93i({Ojtf9>M+k=LH!;7sU0eShHtVFl7*7S-n+1p(-mcF*)d#tvx%i{f^VIdqe*`#lno+%1RQb)_dV? zljaGr$TLTax{|)TN__OfuLaC+oJe<8_!sc+aCD#Hk9#SpFMG=tK0xD=KLGB}*EN0G zHn!_{LRf0lC{YWerQnQpcQ~xl_LY@lPLGq4EUj(vZ`il62w64IMW(cByyf|Iw1mi; z1~{C+F$d!f*5`do%`WN;ctI$@tbR7I;D<}OAfsi=O}`^`igoOZ=A;uGrn)aTl~hIWp>@@cc@aT zx?q6c@JzqYQ9zpepyU)4410fne`;`tr|oi7?%D}J)p|gmwMgFw2%;10ZKSI&5&RYK z0}(Ct7V5MT1zN16EcztTp&ic8q1af@$IgacF&M9|6r?r%Ev@f~@r4t^0Pg`Fi2UwF zr7plk8)5p;o}$tHeWC`Jw#RiqV5yQZw#s-&KK|GmVqOpSje1N}=B#hngi*qRTs^Jr zHiN3uFY3>#z4n0GV~Fj;CQnAoN*%ZS;u*8N4dv3>0(`ALdnaZWS~hd5coe?Uo6+C( zhtox%6VByDsRnGEm9zLJoSYYQSlhk(fAsMi3EJ867OR#rD#u{hnvmt)KBh4Upk}6a z4+=jGxDrdv?(Q(#`CJ?^-%ETdr-u+4gV7F1%2RqYACH?*BMIY7^Q6gt!x=GV2M?&b zHV$9}5oVmZ?UMytYDQTN*m~^96jr1&vE}MB*tRO!@*8eMch|!T6p{gfKpbZQ3Zl&l zQG#V2>fV~#vuh=0PqSM-g&Cdwbrk*3h?SI?n8_028VsyI_m!HR;VYCDOH~p`Irq@B zKYbju6$m&PuyWjy@HCkMU?UR-!p@VuIgI83dJ^YqsRhnuYwBejL9c|-3hT(wE9=d? zA{*jxi%n!rjJl_aZuX|*bm!|4K@ROgVzNRkw*#O?-PTB!iCCTm4<)+` zdl-nvcj_QRW_a9@VQ7y#44&y_eu99b_M!Qs$)_+tu%{AuRxAMKABlPqW+rk*cG#vS z3UiF}keN#CTzYm=<6Lm%?|yzx&CEF7-5EXb{nLQn0CsmFJ@Dgxq$7Jqk0ApDbkJAu z`czs#$P55kgfMnw+wE|Qnl>Xm4Ba@5`We)EkT7VHdJv5>J2$Lzo&NY>vaB*b7#M3a ztdQZDbfWIhFwP@_0T^$i!$duHN2%jyi0##m1#?L#^rCU-@C5bNlsvH^B!luNKW!GObxF?I@x z3$|AW9MW*F32FhWN@>{TNHH!1KfJV@MHwmX#?jyU$T{hKXB4Q>14|JYfgj4|Hjp3Z zl8V9JPtSlUZly>tK^Lkh6)=YT1h5m9qy!Cw*aZ>u24aQv(JKczdIWYCjB&4*bj1Z1 zY5-vmcdVh9`PqlOnD!ta3Z4;-`M8zok@Kws@4I|hI(; zaq?LAZumZ z(Sji^9c52wfT6@e5$s2sBEEh&N67Webc%r8k9dywos(RCavos`!8!H66W4+MqLH5$(VL+0MWsV#_8 z`{XTr76o-JiVP7CZ0VX5TbHv*yol^ZBky_4SgVYgPxzD*srvpsT=Lr9F{4Yg0lF;l zNv)+J4@^|f*5~;-s=`Z;my7(oB#D`N5x#joldQnJbG98+*k#Y-xR6bqHvk{Ku0I`e zr7zW>dB_XbU`%&RrqnRWqQ?=B1*IuvC8Q-&V>hclY<9Yr`Q7Z=82=BoAJa>=x&bf2 zr&v#HXS#l>Ep(G*H*Pr0(G0E$Zhwjbmc})oYF1(ZZJJ3h#ID~>pJvX6@}l=Sp*m0= z6^3Q}@Gc$%-vYTU9#?F?%onvizwz7d1bVR6sWa}0%%m<|U)|3C##mzf*VNsw9NNDZ zjs7=Ncaw; z1^+K|cmE+u{A+sa-;&Uu@Ya7veEFXroPXDQVf_CzdH0V=@|R!ozk2@Tul#xZv0DBz zVE#(P{bye89}|WBAMO1yVE(c1KTMdv%$I-KF#q+F{&HCU_$&XkT>dKi%YgZZ>+0kH6mkuO`f2-|fG3Vg7O5`qv(1{BQDj8UGLSc-_ptRuEsbW?OG4D_PgZF@u(C zt)RuWLlfHxC4?h}`oc%4$ZvBPiK4ea*tg61Bk&VKOE6fz2?*HrVvsO|&6wKTj~hyJ z4=NkFOuD!r-ez5HIQXm`^1eGND=Mj8vUHq!EiIbg$$qO)IXyu~MNiMfXlCb~vAGUM zg)DvVKsHD*t#^+S%`d!+3s=r}T%=?r6IrCdxi8={`{22Kq{EQNw_7n)h(WX|9U8+q zl4tVA<$Q3GCk_zpGi?|zNqQ~}0CRP{OGiamYqQz(Mn+05tO^YemA1z^V=yhzG;LkY z_-uZrmWd1xbvc#bE1-^&QA2BD<~`nY`TX?ylv`a>+FkpopOk61;G?tP&7GVEF>5r_ zG22`SHg7VOs@LVXcaU`co$s(nvru@Nb-#Xh@nZI(_s;z;5Nra*D3pdM4S+mCOfMk~ z>JZSTrY(y_(oLBj-=;ZIJ z9_xYZ-6gQP9^)gcRJ~MV2sUaaOV&LsI7~P!N*FR26WIG;Ihai>Z-zJPCpscVji~j= zON=2*lW!>)`H?k)#6Mc?q8K(^=Upe&Y%G~|*u3@MHl75hAvpbzVFXQbi?;||q4GiK zdc>|!1p^@nM269Bf;#-QOVB2vmjYjV(R&rQxwc!b6iksnjAql=6K3Xnh|}CNz|urh z2UAH?J!QO$_~(EW6Deluf7cJxuPT{RGT|VLMaJA@(13TOOoyaW2f#;DWm})V_B1i+ zXmmXl6g}sy*_OSLAyDf{yW%>IIa^IWA@7!JhN^QrQPRAWXw@~F&e}p_?rcu2uYZDL zzyMeId`7?ub>4J}@8t>$4JI&bB9K-4-(%VOGtfJe9}TbGp!G($n437Be#51Avv-{< z+cd|z9K~>5XHcWH&Wgj+#prR%dZWH8M0NLhg}rm_xPr3#l`Qf2m<9^Yc;=KmR{Tp- zLr&P^yb&-aXX*tpMjV~W?)HvYy9jl>B05)CgzL(Jyb8E3x9AYyW2y@ zvQjJ7QdSs%l{uwKkaLJZi{xSJmK1d%;TbWnJyD&774_M$KYQVLetH%%f%6(bMApD- zB=B;ALjM|9Zk#@0xW&8+AyEDKuBMuR>!P-i9p7dDhy)iQ?31w0$;RP4!)eX?I#bCM zrHN}#r>?K*siX2?4<{WW|5ACj0|q@`Gqq_&>|PbS_gvFhzkCmgRphX&YcKsqqy!1V zi$^fT)GY2`ib~(_6prhk$B4F7{?9ge1br@3=A>Qrzg9&XgBS~d(*-|T z?1z8(rB&6@c9!WyI|@P%*C_483uvi)wjWFiixpo>xo-7;u}Gd%CsRpURZ*$nMzKB|A1(Y5a(bceo8cHk&uR|^c=jJ$k3+H`ibo+dF z8P}zzPZ1ehgYZoGhPSJ^555bwFl+cavrRZ_^pIbbh5+#{&c~Pl68m8YaDB;hpf+Z> zn#hG1(^dF@B%D6eT=ER)prW+d*z?2xO5=DbaY~S{enBK8al9T(%!oFVF5`BRT1SC? z{e;Ki`6~{QhL%cyR~qiF#M|OJE@Cm5=r|(3mJOsTX+1tIEG^gXQms`R^69y5Fuce- zW96`;b#4T{JVh3-gq?Ot1yu>P!S;Yt?2(eebR%<%WQ4@&5E@Q<-J-3%*!J)P7~9k) zBKw@8ti`aR3I?x9s>njCY>z>>eS+#0!P5Th2frya3B>l=40lv0&Ns8~&dUv#w*>8s z2)LnpJ;-6H;uIskv-G&Q)v5Jx+owWLGe|0wgA(i^-V_<$6iCv)vF@;`0JlOR-6LhpK4V0rrjB> z!;)pu(6haTr<+tUq2{&b;?$&8t9KZ2gvQ)ks#@|d>1CQhsna1xCB*c`kU2z`aq4t4nD-Tsm0!XXskMHC@u$* zFv2I{D>TIKlHOhodS?g?>=m2ye^quBP*HVVTLh#_TDn05W`-e#mJVqor5QrHTO>rf zQ@R_ZTN)$>8A_B60qI7GAN2cvUwyyzzu$lFx_8apcb|Rs*=L_SYu)wSXIopXS=^Nj zm8E>_6X4h0$q1!do<<3hlraH5_{zGN7o*a#8_M~nzkJ(440LN6h7m_L4P$phenHOCrJ%ImyFexMC9Rhx*kt^7M zOJCUU92<>25upQBgkU`Fs&t(C8|7IZQgf#N&ONV3F_DJEpT?YymSoNkW(Vdo{gIaf zUt(;S(S4VR=Rj6Nw|h!!aI{s*!fTx5E-s6hL^h55AyxIrTbOlRV z@g2{-6pH4&G!^H%I%`?ejtwK!fF_|na+g1P8<%%FZxp1Cw;u0ARR1aj+_``|YtbO< z*x^NR#(C(@)^gWhA@~&$-T*UiqOzS$wq8@1^bl1RX3#q3k(t2_ji(#jXI?xPw<+}c zrTQAkLc&KY=AE&vRgyT}kx`N^DW(TGv+iOC;?bJp_B%DB>w_HDEz$ZS9K|9bue@)7 zD-J4J0B)@G{&)V(cQ^0m>iX=GNpSUgSoK^my>+zLuX$|@ zW-+>(i5fbK`dyJ8*Nk8vo`2bs(7QP!2krT)f-`*r-FniGv#6IdTwl!!w`|n;0JrSb zMB^We-IH@=Oz<3ZyriPWBwgRAZRU8RG7=a+*i#8?zFV5Jr|+(8(`vtVRC}1Dt9RTV zBl>mcLf6y9dM@2wDcuKDDzt>HQROmNhc#I$pQ3IPmFU6o24YE*7~kq{TnE*CF-iGI zVaZ;H{E*V`MTGdi=KfU--(Xr)C_Yo*P;!fjKJBC`k_LZK6r-TnapazeHqeqo)=n|N z+>h}CZWUGCg{(TjK+J^1>#^8_Z#ncq-C->@GD&k*eS`wyk9$)(9h5OJ)+(_+g`5(8 zD#nQ?9M}9BYTP%-yKEw%o^||eb>sDSC=K0vKF-Fy`{`0kdTsev?+`@eD13rP;^;Ya z8))r)1&9`}25g#%TJCHtn8cup#d!phM8EDTYg_7#ia-s72$=*?QLE!NOUYuCyDK5}P*yjX(>FffnWBrk`7 zmFaefd@b!49hqAzKzgo3n?ot9i5VFhiCPgjiAPC+FQOWpwhzV`CM_xpuF~1WyjeOCd1MDL39FI8 z?Tc&sy_z^u$cU2Sy26Puajwz;G=IYtiX@!S4*b9_QB4tnj@U<0`@1`W6NT`u@waaC znxq_2HJW+aDYRWg+KJtXX@%@n8SfM8cLs+AZM{1f%ln6QIF@bU?WHSQ@p~!i3VKx! z+nYmcNSlmjHZadLhnD=#)jK`!CtzkMt?P2>oCqsZxem^JTin2>z>ImZZFHDbw9xk8a7*VpMNwasdy0O{Z4cGv^C28U2 z($n_QnQ?U45&Nn^%yCsCMb!RGVmm~A3O^zumTZX7iQEQ&om-&cQS7GY>*zdq^(c_)hCTZU*x$52MimZZABYS z1#~()xfVaFLbldFWQ+6CdTA|ACIi|mFo;_DnX>m*(A{gj#TLKu5&FL^Jlb;>-^#Yg z8TyQwM!>Y5i|n&5&M(6>Q^L2g=Oq*TlJ^7NV|new`y5bZAg+|*BYWqQ{Nt~X)Il-c zo(CuL11g&82}eRQIIS4-e9;?txil1$|8d4H;ubpg85K$Lq zUdc;2Oxv8yC=SmyzCTj7R>TsCn_!kTrs0VmZ|{}Nbw%_}R|VBq1ibUYmjoJh?ceI^tAwvKZ2i_`2hpWS{- zdDTE4xljYmWl~!KRI~KD)QL>p3)SHaS6xtb=n?Fphkr7Oq)J6(w(`q>6zwO?abvKE z{=!=DO#ZV0R>?Y)Rpl<$ft4pwk8!NC;@*z+nZVPv`tFK&rjTAkZo6UKBF(8xzh`^9W~W>y%I2 zr;&}DPl>_3^%R4FD(XB0ra0maWYG8I@}EWdg?wmvTzB=#OE>T2yJuVF2h)OA1i!ps z0K0#p(V;_r6i%{0mO((dPE})NKiyrkQi!xm2*mfwhoPqn$mft|C+8+-11-LpUvO1} zW4Xk*C?6;eV}{rFO2Jh*b?|fDH8^$Q^7w_x#T(@tS){q7ee&9$Tta_Obk95hN3mT{M~c0c=TWo^Jkb(yPS1$O6s_z8-r}V`|M|O*JM9 z`NT(A^?k3?O$LLHE$UvokTa?&U@ znmC$IafWwIIv1gWzl;^!qd5Ctsq9Hp=ZGw)yL9E4WD53D#^sna-!T8TUn;VoA!EJjh0SgAYc6 z)o^bkZ9lPoQz!F1I(FEiP&G+44(?uHT42T}+4mJGErVS{qs%i~^(gH~>l?fll|*Lz6ESBM2i!BgrFVX{>1+d^~*fPVbg4I?rnlefIg%S7*#H!@R@A!qR(z zlD5kx%WBKdjB9|y4I>R5NtT~&1)d8)OL_D->qvCMal-Y<+IX&vE{3N~FB01^%0)G@ z5*KJNi6t@SI!$Oqk~qZCgFBzo=uES35u;!Gm$ZlNSG@n!?QF%fB%kj!iO8p4%CDqs z?U@7b*CM%UE3r3l9-u}J{wmxrK&jjj-#a1<7~G_2p`}+C3DTz`wITI#=O&4@s@KrS zXVT1GQe9IF<*=AGq7xW66~grf1%?WLh_W9>@12Yk(T~@C*QBXXe$2n-9CbLGsHACA z#LmL&TEtA}Mr@6M`&uDJ4^tCYD?0)AY3^ciuD!&llm&q>jWgzk7~W7CPr-52(04-| zQaVQ2hx-qXgtgWf)^1(|g1%@@Sr5%N=`nh^4DP2jqLd{LoWy-x?TiuIr9v4p%nR8J z!j9ktGh5BMWxV}n#?5fS0*i@O{H83kx_{=mS9vMn8+gTo+>~ltuV3>*Dds_)*YIXQ zxWg&fRD8{=4I$y(@s&PT+FPEq9Dx%mC1#c0rU9SfD#F2rGbpemL9_Ynn&Mu^s1C;q z>qY^yB_Yx|(T~Skt1Pvj6Az^_wyhb2#1%baC+P8Na&a&{-QI0|cuIPjIAfvf4?E=z zC@>njGhrYZSx+wAPxEAM#X8zk6tey-K;^`WFV2w@N3x>iq-^mGYsni*)tHq_r8Hsh zLiE~Z+95v#D`V)m+I&NUD^$R^qouttZVdiq`*p=mb=th{+rtF}?0Um^)IIZ<95_K9 zzL+L>;)6xZHyIHF6$o48&ciX?*SRlLnu8MqbZIb#x|HCxAqDMPA}M>3nWj3+srwT- zWjTFPC?nDS7x)4}bD~P;dnIt|uEL#6eG$D?!^rlDyWZtKr-!g96eYBeK&t=^Kdku# zmyERUdB-If#6B~u7z|!DzH6JPb6)&1AWy3(yRCzxw*a~JWQM81 zP#1!Uk01_^WnzKVP))w+bfl5bMsk24a~9DAgo+U-7=f}5cAmhS{RP)W^<-><1!8Nf zotT(dwl=VsTGsf+KYCK+FvZ$g_Qi&gi@<-Iwt*7Qw4p}3;u zCgs+Aax*j%tEr0YFsX&Bv5~BFZKi!XAX@nVr?kFEN}NTyglO7Z^UVCpAAkWY3F% z$jdyH?Kv~{of}GS48^!xntQ0UB;FeV!F4hU+!C&P_^v1(n1La9`a%+((tkb;Q|A)b ziHCT*#^}4`ju{69XjSfvVMq6{ zKFH(fb2H4C-h5B_xJB>le2LlF_{OdT_d6u(<9MB_Or(c|t?pzlYDx&lggyqlbzYW> zzIRx5Xcd#my?2;BbTQ3dmt#kUDzsiLb)^-YPE4P~?Xnd<$~fAd7w|Uf!{xys^(^}i z+npj8(tVD>Dwyst`K{%gAq=^G8YZ~gAA1jYT#d1IRMk-MO`@kv=zL=5^QAj7_v3)J zthG0&KG*M2eUdM*77b*H*hPX-XBY2?mA+5GB*#IVsN{8`Lt$o&2Td+Zn8#MDlE0ur8eA$gl)>>cjR_?(=Rr7pLh1K z3}zdv49p2THf$w@v2MvI1B+zP?2CFP`nIJWnv_~6b(SEGy;B<{IEb*wtv+LeeAnE6 z6SQU%p+#w+8_JwoXpB;NWNu;cVG+(C(Z`|nF2yP5Ye4&1rnHXi__s4GIAlTTaBeB5 zpp0oS)+nLCtlKKCzEmfL%+}d#9yUDuk#1SX?4gbNfU)M2CoBR&X=2H4=HH`?@1(w* zdwVR^A07a>`v60b^x30f@zzhTj$<*3-}ZJ}#PIF*LTJ{rDF}nH4f+%e$ zx-S#&6Kwc<_qWyf6j=I5^JZ+7DZH5QV6Fi3NWzcLpcSipia}!)ZQ{5*BFiU0Wh*!b zA+H_2@ZezwtsgG+7(ILJ9;eV59w)B%R5<{)lMq+2?N$!$j&Fxi>DhQW*+UtQi9Spl zVxUH$GnRTsHhFjB2HHK69!*0)lpE)~B!lapxcSo66%fYhM8s1;+_h(Ut1dm4*FDCM zHY+E~WAm|4Swi^Iq-m%FHaTSdIxw#BWT_OiX|R#3FDv*+?!dY!#WO>jg6oi=jB{V8 zrY*1TvjAd~ddSQNONZ!>`6PL0>16Mgi5J)qhg0M4;8Uzmgd?pACCzQoGQ-6Cs+nSp zwa_aBjFH{k!XAK_oNNpXW}jKTjyU=<$~1&E+tTGA&= zEVJNk(jz)Q5Jc>YIJ=iY(K(;fR_r1ef;7LA?-AWk6ze_7rk~a=k;e#qR6T;)df0W? z#6Kleipj8LPg)=i*u25xcfl%ZoyWU%)g%%S!>xqhtCTb{p_wFzW|E;{itXS|>n&VL z6LI@UIp3R0tr9ZJb5;>O7bLx4wi_`H{wk=yuKkwZ)skivZ2vlsDCD)2`-MHLCWgt= z>N+EaY~)z12!+kLJ5K_8dj$f)$}Atd^@b6s#@;I(VlcLwfZx~w{W)bgc4%hM5@z(@ zA_fnN)aS6m{bB9YWBUT zdblt^6Ih&QBt6yP;>I51w1mDvfLGVn%v5IuNmO}OTi5%|UF>iQ%VI(fyI->?s9GEiWxO9@`P|U=enJTQRb5p0=GrM1ZIGkelrd6#;QudV|Mc;6(CH#)4O-2U+qAx;TvFUE{t#eci|;_DM#uc?<)ov@GWX z0te_2gRPGBavUI^13H8gFJtEjcN``)!Tc}$lkX5=ccMGs-cKgAas>Ei5KfVMnzt&G zQSdrih`($J;7yZsvvDw52g^lH%48yZiQM&7Gw2#V9(tNUo*Q>StwN#?7ygXo7D&5!*2|Phf5_9JzVwKbA z?kQ86&wENJCC6eC0*J}qlO~dEDqhC^hNcgzG#jZOQq;#Iy{SkE>0`Wk%@L2PFNGvj z$82dlVoFX#!ELs)|5lYLGUI%~Ydkp*#IBy=>7im|#Am(pN=geGRLr1m8paQNgTDbgDt5@`5)Aykq4%NtKyCk8xUv zSYhMV*iXjZh|DUG=JAmTKRdI_n?=rOE-7s;>1t#fC?;Z4XSyGP!!8YVF|cNjEGpFs z^tEL8%-|>L*HT&A;^ywOUYv_O{t;{3^aREOr9?Tk>iX)yQsiZ2J8w$r>!>f%M_fa> z@ZS3(h*X5Xa1Q5)eXTZxlYOujF4`TELZJ_zNy)0!DtOenzrTnVzhBEmSVL9%JOw)S zP=VN~W^dTC(7!by_Jxs_CSK>}_!PyHn%G=l)+_z}{cXGd0F}B04g3o##S7&AQIP!h z67*NH)PIw0{{M)7e<7tJr$?oKLrSS(f>zq+U8$5G9^c{DgSnZtDkuy);s-uu+O>Bu z({vU_qB2Y_|KPi`078miox!nbEj!!obx}tPfqkm#oDO)Q@-#0N=kv44hK$6iEkBSbR(|rsA-fpT*)Siqk+jDGR%tU136YY{ke(!49RzB%F`)oQ2 zso<(Dzs5U4a8R+o{Iq|bCQ)&dS%Ag(FMvoGQdW+V9XIE0f;H0iXFRp5{M?=bx#8|2aK`+u z@}uYfk{$T(sH9&K5a>1^{+pc(0;2xk1AM#sxjAp!{<6T$dyBZbm2&;K|3mw?Upzdw z5U)RX@C~{OK>h0s;<@Dj{!y>|hj%~k+_C`QR5SnAal;f0yq%L@x<4)ajV1WUbLy`% zd29FI2+yBbHzh+esN;`-%2}BFDCmBpqVlhl6SH-`sdNuS|3x#*dF%hbEhstInm8Ln z9jKX&4J{mOse$Z3Zgv1Qv$>O#og*Iy#}BEQgQ1Mt%DiM&(s%lfEqek*xHC2 zIzg$K#rZ%05EuXkaRInFIk|YXS^nDJ-#r#HbTYKEHT&W75AUge(j5PC(#-*CihdI$ z5IW$0H)?J!E>13LQ|g~GFedFRTyNL$Z&9Hvc0|0q$6XSOo1agb#{ZsZgdpA7e zTz~d)BfACi{ni$E^T7F=9^}TEKlLCWocNH;XzzmE+J*5 z{J@;RzxwRtU}#|lbwIzxma1BKKyP#M=D&wS#n#s8W)a^mwIA0qHm0`JKi1Zd)#~VE W=-~7ti@}^AP6#>!gM^|a`u_mn*@*rC From 6857b0e4a1149d94a7e92b0cef89b0f419205968 Mon Sep 17 00:00:00 2001 From: Zarquan Date: Mon, 28 Oct 2024 06:36:30 +0000 Subject: [PATCH 7/8] Restructuring in progress .... --- ExecutionBroker.tex | 493 ++++++++++++++++++++++++++------------------ 1 file changed, 291 insertions(+), 202 deletions(-) diff --git a/ExecutionBroker.tex b/ExecutionBroker.tex index 6dbd03e..23eb209 100644 --- a/ExecutionBroker.tex +++ b/ExecutionBroker.tex @@ -9,6 +9,8 @@ % Using non-breaking space character. % https://stackoverflow.com/a/1012891 +\usepackage[super]{nth} + \newcommand{\xml} {XML} \newcommand{\json} {JSON} \newcommand{\yaml} {YAML} @@ -27,20 +29,22 @@ \newcommand{\vospace} {VOSpace} \newcommand{\execworkerclass} {**ExecutionWorker**} -\newcommand{\execbrokerclass} {ExecutionBroker} -\newcommand{\execbrokerservice}[1] {ExecutionBroker~service#1} +\newcommand{\execbrokerclass} {\textit{ExecutionBroker}} +\newcommand{\execbrokerservice}[1] {\textit{ExecutionBroker~service#1}} \newcommand{\execoffer}[1] {\textit{ExecutionBroker~offer#1}} \newcommand{\execofferset}[1] {\textit{ExecutionBroker~offerset#1}} \newcommand{\execsession}[1] {\textit{ExecutionBroker~session#1}} -\newcommand{\executionbroker} {Execution~Broker} -\newcommand{\executionplanning} {Execution~Planning} +\newcommand{\executionbroker} {\textit{Execution~Broker}} +\newcommand{\executionplanning} {\textit{Execution~Planning}} \newcommand{\executable} {\textit{executable}} \newcommand{\executablething}[1] {\textit{executable~thing#1}} \newcommand{\excutabletask} {\textit{executable} task} +\newcommand{\metadoc} [1]{\textit{metadata document#1}} + %\newcommand{\execoffer}[1] {\textit{offer#1}} \newcommand{\workerjob}[1] {\textit{session#1}} \newcommand{\teardown} {tear-down} @@ -85,6 +89,9 @@ \newcommand{\datascience} {data~science} \newcommand{\scienceplatform}[1] {science~platform#1} +\newcommand{\science}[1] {science#1} +\newcommand{\scientist}[1] {scientist#1} + \newcommand{\cpu}[1] {CPU#1} \newcommand{\gpu}[1] {GPU#1} \newcommand{\nvidiagpu} {NVIDIA~AD104~GPU} @@ -316,7 +323,8 @@ \subsection{Executable things} they need as a single binary object that interfaces with a standard execution environment, referred to as the \textit{container runtime}. To be able to use this \executablething{}, you would need a computing resource with the appropriate -hardware and software environment. In this case, a computing resource with the \dockerruntime{} installed. +hardware and software environment. In this case, a computing resource with the \docker{} or \ocicontainer{} +runtime installed. We could also create a \jupyternotebook{} that demonstrates how to use our \pythonprogram{}. This is the third \executablething{} in our example. @@ -340,7 +348,7 @@ \subsection{Executable things} resource will probably be sufficient. However, if we have a \dataset{} of ten million numbers that we want to process, then we may need to consider adding extra storage to handle the input data and the results. -For a large \dataset{} it may also be worth using a \gpu{} to accelerate the calculation steps. +For a large \dataset{} it may also be worth using a \gpu{} to accelerate the calculation. The \ivoa{} \executionbroker{} \datamodel{} provides a way to describe what each of these \executablething{s} are and what resources are needed to execute them. @@ -364,8 +372,7 @@ \subsection{Discovery services} \includegraphics[width=0.9\textwidth]{diagrams/data-discovery.pdf} The detailed specification for the software and data discovery services are beyond the -scope of this document. -However we can outline some general requirements for them. +scope of this document. However we can outline some general requirements for them. In both cases, the discovery process should not depend on the technical details of the software or the \dataset{s}, but on their science domain functionality and properties. @@ -377,10 +384,10 @@ \subsection{Discovery services} are at best secondary criteria. In our square root example, we would expect our user to use search terms like \textit{'square root'} or \textit{'newton raphson'} to find the software they need. -We wouldn't expect them to start out looking for a \textit{'python'} program or a \textit{'docker'} container. -Ideally, if the \executionbroker{} services function as intended, a science user should not +We wouldn't expect them to start out looking for a \textit{'python'} or \textit{'docker'} as their key search terms. +Ideally, if the \executionbroker{} service functions as intended, a science user should not need to know about programming languages, software packaging or file formats. -The \executionbroker{} services should make all of the technical details invisible, +The \executionbroker{} service should hide as much as possible of the technical details, enabling the science user to get on with science. Another important consideration is these discovery services should be designed to be domain agnostic. @@ -389,24 +396,106 @@ \subsection{Discovery services} terms and vocabulary will be different, the techical details of the service interfaces should be the same. +\subsubsection{Software discovery} +\label{software-discovery} + +There are three main components involved in software discovery, the metadata schema for +describing the software, one or more search services, and the +repositories where the \executablething{s} are stored. + +The vocabularies and schema need to be based on use cases that start by describing what the +\scientist{} wants to do, and from that derrive what software tools they would need, and what terms +they would naturally use to describe them. + +The \ivoa{} semantics and data modelling working groups have a lot of experience developing +vocabularies and data models to descibe \science{} data products, and is well placed +to develop the vocabularies needed to descibe astronomy software. + +It is important to keep in mind that the requirement is not to model the technical properties +of the software itself e.g. what programming language it is written in or who funded the development. +The important things to model are the search terms that a \scientist{} is most likley to use to try to +find the software they need. + +The second component is a searchable database that acceopts a list of search terms and responds with a +list of \metadoc{s} that describe \executablething{s} that match the criteria. + +Before we look in detail at the content of the \metadoc{s} it is worth looking at where the \metadoc{s} are +stored in relation to the search service and the repositories where the \executablething{s} are stored. + +In one scenario, all of the components can be co-located by the same service. +The database of search terms, the \metadoc{s}, and the binary files containing the \executablething{s} +can all be hosted by the same service implementation. + +TODO diagram + +An alternative implementation could store them at different locations, using +existing off-the-shelf software and services to host them. + +There are a number of widely available content managment systems, both commercial +and open source, that would be capable of implementing the database of search terms. + +If the \metadoc{s} are stored in the same database, then the response from a +database search could contain the \metadoc{s} themselves. + +TODO diagram +database, results, contain \metadoc{s} from database + +Alternatively, the \metadoc{s} could be stored at a separate location, +in an online git repository for example, +and the database search response simply contains a list of URLs that +point to the individual \metadoc{s}. + +TODO diagram +database, results, links to \metadoc{s} in external repositories + +The third part of the set is the binary image of the \executablething{}. +In most cases it would probably make sense for the \metadoc{} to reference +the \executablething{} as a binary file stored in an external repository +rather than trying to include the \executablething{} as a binary blob in +the database. + +TODO diagram +database, results, links to \metadoc{s} with links to images + +The system can use standard cryptographic signatures and checksums to ensure the validity +of the \metadoc{s} and the binary images they refer to even when they are stored and accesed +via external \nth{3} party services. + +In summary, there are two things that need to be standardised for a software discovery service: +\begin{itemize} + \item The inputs to the discovery service, including the metadata vocabularies + used to describe the software in terms that make sense to the \scientist{} + looking for them. For example what algorithm it implements, the type of input data it + operates on, and the type of results it generates. + \item The outputs of the discovery service, including the \metadoc{s} defined by this + specification, that describe the binary images that package the software + as \executablething{s}. +\end{itemize} + +The other components in the software discovery stack, the database of search terms, and +storage and access services for the \metadoc{s} and binary images, do not need to be +standardised at this stage. + +\subsubsection{Data discovery} +\label{data-discovery} +TODO : update this with reference to \ivoa{} data product type. +https://www.ivoa.net/rdf/product-type/2024-05-19/product-type.html -%move this to a different section ... -\subsection{Metadata roles} +\subsubsection{Metadata roles} \label{metadata-roles} -The full description of an \executablething{} consists several layers of metadata, -each of which may be provided by different actors playing different roles within the community. +The full description of an \executablething{} may include several layers of metadata +provided by different actors playing different roles within the publishing process. -For our square root \pythonprogram{} example we can identify a number of roles that each provide -part of the picture nedeed to fully describe the task. +For our square root example we can identify a number of roles that would each provide +layers of the picture nedeed to fully describe an \executablething{}. The players: \begin{itemize} - \item The developer - The person who wrote the program - \item The packager - The person who packaged it in a container - \item The data provider - The data provider who publishes the data - \item The user - The person who is running the analysis + \item The developer - The person who wrote the program + \item The packager - The person who packaged it in a container + \item The user agent - The person who wants to use the software \end{itemize} \subsubsection{The software developer} @@ -420,151 +509,191 @@ \subsubsection{The software developer} and a list of the \python{} libraries that the program relies on. \begin{lstlisting}[] -# ExecutionBroker service response. executable: type: uri:python-program requirements: - numpi: "" - astropy: ">= 6.1" - \end{lstlisting} The developer also understands how much memory their program needs, whether it can make use of multiple cpu cores, and whether it can make use of a \gpu accelerator. -So they can provide an initial description of the compute resources -their program requires. - \begin{lstlisting}[] -compute: -- type: uri:generic-compute - cores: - memory: - extras: - - type: uri:gpu-accellerator +resources: + compute: + - type: uri:generic-compute + cores: + requested: + min: 4 + memory: + requested: + min: 16 + units: GiB .... \end{lstlisting} -They also know what inputs the program expects to receive and how it outputs the result. -If it is being used to process a list of numbers, the developer knows where their program expects to find the input data, -what file formats can it accept for the input, where it puts the results, and what output formats can it generate. - -\begin{lstlisting}[] -compute: - volumes: - - path: /... - mode: readonly - resource: input data -\end{lstlisting} +The developers also know about what inputs and outputs the program expects and what file +formats can it can handle. \begin{lstlisting}[] -data: -- type: uri:data-descriptor - name: input data - mode: readonly - data-type: List - data-formats: - - type: uri:dataformat-csv - .... - - type: uri:dataformat-tsv - .... - - type: uri:votable - .... -- type: uri:data-descriptor - name: output data - mode: readwrite - data-type: List - data-formats: - - type: uri:dataformat-csv - .... - - type: uri:dataformat-tsv - .... - - type: uri:votable - .... +executable: + type: uri:python-program + .... + parameters: + - type: uri:param-file + name: "input data" + mode: readonly + description: + A table containing a list of numbers to be processed, formatted as + comma separated text (CSV) or an IVOA VOTable. + formats: + - type: uri:text-csv + .... + - type: uri:votable + .... + - type: uri:param-value + name: "input column name" + type: string + description: + The column name within the 'input data' to use. \end{lstlisting} -ZRQ -Question - what is the relationship between data-descriptor and data-resource ? -Are they parts of the same thing, or are they two different things ? -Do we replace the data-descriptor with a data-resource, or do we just extend it ? - \subsubsection{The software packager} \label{software-packager} -Packager changes the type of \executablething{} from a \pythonprogram{} -to a \dockercontainer{}. +Although it is possible to publish our square root example as a stand alone \pythonprogram{}, +it is not easy to describe the installation process in sufficient detail for it to be +automatically deployed on a range of different platforms. + +A more portable solution would be to package the \pythonprogram{} in a \dockercontainer{}, +installing and configuring the software along with all of its dependencies inside the container. + +This is often done as part of the software development process, but it is a separate +step that could be implemented by a different person. +To make this distinction clear we can refer to this person, or role, as 'the packager'. + +In terms of the \metadoc{}, the packager changes the description of the \executablething{} +from a \pythonprogram{} to a \dockercontainer{}. \begin{lstlisting}[] executable: type: uri:docker-container + repository: ghcr.io + image: ivoa/calycopis/java-builder + tag: 2024.08.30 .... \end{lstlisting} -Depending on how they package the software they will probably need to change the -description of where the \executablething{} expects to find its -inputs and outputs. +Depending on how the software is packaged in the container they may also need to update +the description of the inputs and outputs, +and link them to specific locations in the filesystem. \begin{lstlisting}[] -compute: - volumes: - - .... -\end{lstlisting} +executable: + type: uri:docker-container + .... + parameters: + - type: uri:data-file + name: "input data" + format: + - type: urn:ivoa-votable + filename: input-data.vot + .... -\subsubsection{The data provider} -\label{data-provider} +resources: + compute: + - type: uri:generic-compute + volumes: + - type: uri:file-mount + parameter: "input data" + filepath: /data + mode: readonly + .... +\end{lstlisting} -The data provider describes what the data contains and what formats it is available in. -This can be used to match up data sets with input and outputs of an \executablething{}. +\subsubsection{The client} +\label{user-metadata} -Beyond the scope of this specification, the schemas and vocabularies needed for this -level of metadata is the domain of the semantics working group. +The user, or the user's client agent, starts with the initial \metadoc{} from the +software discovery service, and adds additional information describing how the user +wants to use the software. -At this stage it is probably not practical to try to define a universal set of data -structures for all of the data across all of the \ivoa{}. +Adding details of the data resources the user wants to use enables the \execbrokerservice{} +to transfer the data to local storage before the \execsession{} is started. -However, it may be possible to start with an initial set of data types for -individual projects and gradually work towards a wider concensus. +Including a value for the filesize enables the \execbrokerservice{} to estimate +how much local storage it will need to allocate +and how much time will be needed to transfer the data. +The \execbrokerservice{} takes this into account when calculating the start time of +the \execoffer{s} it makes, allowing enough time for the data transfers to complete +before the \execsession{} starts. -Examples +\begin{lstlisting}[] +resources: + data: + - type: uri:simple-data-resource + name: "input data" + location: http:data.example.org/.... + filesize: + value: 145 + units: MiB + .... +\end{lstlisting} - uri:eMerlin-visibility-data-001 - uri:lofar-visibility-data-001 - uri:lofar-visibility-data-001 - ... +Linking the data resources with volumes on the corresponding compute resources enables +the \execbrokerservice{} to mount the data resources at the correct location in +the compute resource's filesystem. -Even just applying a URI to identify the data format will enable -basic tools to identify which data can be used with which software. +\begin{lstlisting}[] +resources: + data: + - type: uri:simple-data-resource + name: input-data + .... + compute: + - type: uri:generic-compute + .... + volumes: + - resource: input-data + path: /data + mode: "ro" -\subsubsection{The user} -\label{user-input} +\end{lstlisting} -The user provides the last step in the metadata. -By selecting the data they want to use as inouts, and selecting what format they want the output -to be in, they can refine the links between the -data resources, data descriptions and volume mounts. +The user can also update the compute resource requirements set by the software developer +to provide additional details based on how they plan to use the software. -In most cases the user will use the -compute resource requiremenst set by the software developer, -but they can provide extra information -based on their knowledge of what they plan to use the software for. +In the case of our square root example, the compute resource requirements set by the +developers will reflect the original intent of simply demonstrating how to use the +\pythonprogram{}. -If the original software was packaged as an example notebook, with -fairly low requirements for compute resources, but the user intends to use the software -to analyse a much larger data set they can modify the compute resource requirements -to match their use case. +However, if the user intends to use the software to analyse a much larger \dataset{} +they can update the compute resource requirements to match their use case. \begin{lstlisting}[] -compute: -- type: uri:generic-compute - cores: - memory: +resources: + compute: + - type: uri:generic-compute + cores: + requested: + # min: 4 + min: 32 + memory: + requested: + # min: 16 + min: 128 + units: GiB \end{lstlisting} - +TODO user provides the schedule ... when they want to run it. \subsection{Execution broker} \label{execution-broker-intro} +\subsubsection{OfferSet request} +\label{offerset-request} + Once the user has selected the \executablething{} they want to use and the data they want to apply it to, the client combines this information to create a complete description of the \execsession{} the user wants to execute, including @@ -572,6 +701,7 @@ \subsection{Execution broker} and a schedule describing when the user wants it to run. \begin{lstlisting}[] +# ExecutionBroker OfferSet request. executable: .... resources: @@ -580,11 +710,8 @@ \subsection{Execution broker} .... \end{lstlisting} -\subsubsection{Request for offers} -\label{offers-request} - -The client sends the \execsession{} description to one or more \execbrokerclass{} -services to see if they can meet the requirements. +The client sends the \metadoc{} description to one or more \execbrokerclass{} +services to ask if they can meet the requirements and execute the \execsession{}. Each \execbrokerservice{} evaluates the request and responds with a top level \codeword{YES|NO} answer, and if @@ -602,23 +729,22 @@ \subsubsection{Request for offers} \subsubsection{Offerset response} \label{offerset-response} -The \execbrokerservice{} will respond to a request for offers with an \execofferset{} +Each \execbrokerservice{} will respond to a request for offers with an \execofferset{} containing some metadata about the \execofferset{} itself, and a list of \execoffer{}s -describing how the requested session could be executed. +describing how the requested \execsession{} could be executed. Each \execoffer{} in the list contains some metadata about the \execoffer{} itself, -including it's identifier and expiry time, followed by details of how and when the +including its UUID identifier and expiry time, followed by details of how and when the \execsession{} would be executed. \begin{lstlisting}[] -# ExecutionBroker offerset response. -result: 'YES' -state: 'OFFERED' -expires: "2023-09-18T07:05:21" +# ExecutionBroker OfferSet response. +result: YES .... offers: -- ident: "2e164a1b-7ff6-11ef-8412-4bc36fe2face" +- uuid: "2e164a1b-7ff6-11ef-8412-4bc36fe2face" + href: "http://..../sessions/2e164a1b-7ff6-11ef-8412-4bc36fe2face" state: 'OFFERED' expires: "2023-09-18T07:05:21" .... @@ -629,7 +755,8 @@ \subsubsection{Offerset response} schedule: .... -- ident: "2e16bf4c-7ff6-11ef-8412-4bc36fe2face" +- uuid: "2e16bf4c-7ff6-11ef-8412-4bc36fe2face" + href: "http://..../sessions/2e16bf4c-7ff6-11ef-8412-4bc36fe2face" status: 'OFFERED' expires: "2023-09-18T07:05:21" .... @@ -642,36 +769,33 @@ \subsubsection{Offerset response} \end{lstlisting} The user can choose to accept one of the \execoffer{s} from the list that best -fits their requirements, or they can reject all of the \execoffer{s} in the -\execofferset{} and make a new request with different criteria. +fits their requirements, or they can reject the \execoffer{s} and make a new +request with different criteria. Each of the \execoffer{s} in the \execofferset{} represent a temporary reservation for the resources listed in the \execoffer{}. This means these resources will not available to other users while the \execoffer{s} are still valid. -If the user does nothing, then the \execbrokerservice{} will automatically -update the \codeword{state} of each of the \execoffer{s} to \codeword{EXPIRED} -when their expiry time is reached and release the resources associated with them. +If the user does nothing, then \codeword{state} of each of the \execoffer{s} will +automatically be updated to \codeword{EXPIRED}, and their associated resources will +be released, when their expiry time is reached. If the user accepts one of the \execoffer{s} in the \execofferset{} by updating the \codeword{state} to \codeword{ACCEPTED}, the \execbrokerservice{} SHOULD update the \codeword{state} of the other \execoffer{s} in the \execofferset{} to \codeword{REJECTED} and release the associated resources. -[accept/reject/expire state-transition diagram ?] - -\subsubsection{Execution state} -\label{execution-state} +TODO [accept/reject/expire state-transition diagram ] -Once the offer of anf \execsession{} has been accepted, the \execbrokerservice{} -will begin the process of executing it. +\subsubsection{Update options} +\label{update-options} Each \execsession{} has a unique URL that the client can use to monitor and update its state. The \execbrokerservice{} response for an \execsession{} includes a list of options -that the user may to perform on the \execsession{}. +that the user may use to update or modify the \execsession{}. Which options are available will depend on the current \codeword{state} of the \execsession{} and the identity and permissions of the authenticated user. @@ -680,7 +804,8 @@ \subsubsection{Execution state} the \execsession{} to \codeword{ACCEPTED} or \codeword{REJECTED}. \begin{lstlisting}[] -ident: "2e164a1b-7ff6-11ef-8412-4bc36fe2face" +uuid: "2e164a1b-7ff6-11ef-8412-4bc36fe2face" +href: "http://..../sessions/2e164a1b-7ff6-11ef-8412-4bc36fe2face" state: 'OFFERED' expires: "2023-09-18T07:05:21" .... @@ -698,7 +823,8 @@ \subsubsection{Execution state} the execution. \begin{lstlisting}[] -ident: "2e164a1b-7ff6-11ef-8412-4bc36fe2face" +uuid: "2e164a1b-7ff6-11ef-8412-4bc36fe2face" +href: "http://..../sessions/2e164a1b-7ff6-11ef-8412-4bc36fe2face" state: 'ACCEPTED' expires: "2023-09-18T07:05:21" .... @@ -711,80 +837,43 @@ \subsubsection{Execution state} \end{lstlisting} This pattern of using a dynamic list of options enables an \execbrokerservice{} -to change what an individual user is able to do and communicate that to the -client over the lifetime of an \execsession{}. - - - - - - - - - -How the \execbrokerclass{} and \execworkerclass{} services are linked is up to the implementation architecture. -One implementation may package both service interfaces into a single web-application. - -\includegraphics[width=0.9\textwidth]{diagrams/single-webapp.pdf} - -Another implementation may package each of them as separate micro-services, with one \execbrokerclass{} -service providing the front-end for multiple \execworkerclass{} services. - -\includegraphics[width=0.9\textwidth]{diagrams/micro-services.pdf} - -\subsection{Execution worker} -\label{execution-worker-desc} - -The client can use the \workerjob{} details in the \execbrokerclass{} response to contact -the \execworkerclass{} service and track the \workerjob{} status. +communicate to the client what operations the user is able to do +over the lifetime of an \execsession{}. -\includegraphics[width=0.9\textwidth]{diagrams/job-status.pdf} +\subsection{Session lifecycle} +\label{session-lifecycle} A \workerjob{} in an \execworkerclass{} service goes through the following stages in its lifecycle. \begin{itemize} + \item \codeword{PROPOSED} An internal state .. not published + \item \codeword{OFFERED} The \workerjob{} is being offered. + \item \codeword{ACCEPTED} The \workerjob{} has been accepted. + \item \codeword{REJECTED} The \workerjob{} offer has been rejected. + \item \codeword{EXPIRED} The \workerjob{} offer has expired. + \item \codeword{WAITING} The \workerjob{} is waiting to activate. + \item \codeword{PREPARING} The resources are being prepared. + \item \codeword{READY} The \workerjob{} is ready to execute. + \item \codeword{RUNNING} The \workerjob{} is executing. + \item \codeword{RELEASING} The resources are being released. + \item \codeword{COMPLETED} The \workerjob{} has completed. + \item \codeword{FAILED} The \workerjob{} has failed. + \item \codeword{CANCELLED} The \workerjob{} has been cancelled. - - PROPOSED - - - OFFERED - - ACCEPTED - - REJECTED - - EXPIRED - - - WAITING - - PREPARING - - READY - - RUNNING - - FINISHING - - COMPLETED - - FAILED - - CANCELLED - - - \item \codeword{PENDING} The \workerjob{} has been created, but the resources have not be prepared yet. - \item \codeword{SETUP} The \execworkerclass{} service is setting up the resources needed to execute the \workerjob{}. - This includes things like staging any \dataset{s} that will be needed locally. - \item \codeword{READY} The \workerjob{} is ready and waiting to start. - \item \codeword{RUNNING} The \workerjob{} is running. - \item \codeword{TEARDOWN} The execution has finished and the \execworkerclass{} service is clearing up the resources that were used. - This includes things like transferring the results to permanent storage and releasing local resources. - \item \codeword{COMPLETED} The \workerjob{} has been completed. - \item \codeword{FAILED} The \workerjob{} failed to complete. - \item \codeword{CANCELLED} The \workerjob{} was cancelled. \end{itemize} When a \execbrokerclass{} creates a \workerjob{} in an \execworkerclass{} service the \workerjob{} starts with the \codeword{phase} set to \codeword{PENDING}. It is up to the \execworkerclass{} to select the right time to change the \workerjob{} -\codeword{phase} to \codeword{SETUP} and start preparing the resources so that +\codeword{phase} from \codeword{WAITING} to \codeword{PEPARING} and begin preparing the resources so that the \workerjob{} is \codeword{READY} in time for the \codeword{starttime} declared in the \execoffer{}. -If it will take 2 hours to transfer the input \dataset{} -from cold storage to live storage co-located with the compute resources, -then the \execworkerclass{} needs to start the \codeword{SETUP} phase at least 2 hours +If it will take 2 hours to transfer the data resources +from archive storage to live storage co-located with the compute resources, +then the \execworkerclass{} needs to start the \codeword{PREPARING} phase at least 2 hours before the \codeword{starttime} declared in the \execoffer{}. Once all the resources are ready, the \execworkerclass{} changes the \workerjob{} From 2d627caa64d0329f9fe9efbc64a54ff12e2e2bc1 Mon Sep 17 00:00:00 2001 From: Zarquan Date: Thu, 7 Nov 2024 14:38:35 +0000 Subject: [PATCH 8/8] Moved matadata roles into section on data model --- ExecutionBroker.tex | 586 ++++++++++++------------------ notes/zrq/20241029-01-min-max.txt | 153 ++++++++ 2 files changed, 395 insertions(+), 344 deletions(-) create mode 100644 notes/zrq/20241029-01-min-max.txt diff --git a/ExecutionBroker.tex b/ExecutionBroker.tex index 23eb209..71733cb 100644 --- a/ExecutionBroker.tex +++ b/ExecutionBroker.tex @@ -482,212 +482,6 @@ \subsubsection{Data discovery} TODO : update this with reference to \ivoa{} data product type. https://www.ivoa.net/rdf/product-type/2024-05-19/product-type.html -\subsubsection{Metadata roles} -\label{metadata-roles} - -The full description of an \executablething{} may include several layers of metadata -provided by different actors playing different roles within the publishing process. - -For our square root example we can identify a number of roles that would each provide -layers of the picture nedeed to fully describe an \executablething{}. - -The players: -\begin{itemize} - \item The developer - The person who wrote the program - \item The packager - The person who packaged it in a container - \item The user agent - The person who wants to use the software -\end{itemize} - -\subsubsection{The software developer} -\label{software-developer} - -The first layer of metadata comes from the person who wrote the \pythonprogram{}. -They have detailed knowledge of what the software does, what execution environment it needs, -and what the inputs and outputs are. - -For the square root example, it is a \pythonprogram{} which needs a platform with the \python{} runtime installed, -and a list of the \python{} libraries that the program relies on. - -\begin{lstlisting}[] -executable: - type: uri:python-program - requirements: - - numpi: "" - - astropy: ">= 6.1" -\end{lstlisting} - -The developer also understands how much memory their program needs, whether it can make use of multiple cpu cores, -and whether it can make use of a \gpu accelerator. - -\begin{lstlisting}[] -resources: - compute: - - type: uri:generic-compute - cores: - requested: - min: 4 - memory: - requested: - min: 16 - units: GiB - .... -\end{lstlisting} - -The developers also know about what inputs and outputs the program expects and what file -formats can it can handle. - -\begin{lstlisting}[] -executable: - type: uri:python-program - .... - parameters: - - type: uri:param-file - name: "input data" - mode: readonly - description: - A table containing a list of numbers to be processed, formatted as - comma separated text (CSV) or an IVOA VOTable. - formats: - - type: uri:text-csv - .... - - type: uri:votable - .... - - type: uri:param-value - name: "input column name" - type: string - description: - The column name within the 'input data' to use. -\end{lstlisting} - -\subsubsection{The software packager} -\label{software-packager} - -Although it is possible to publish our square root example as a stand alone \pythonprogram{}, -it is not easy to describe the installation process in sufficient detail for it to be -automatically deployed on a range of different platforms. - -A more portable solution would be to package the \pythonprogram{} in a \dockercontainer{}, -installing and configuring the software along with all of its dependencies inside the container. - -This is often done as part of the software development process, but it is a separate -step that could be implemented by a different person. -To make this distinction clear we can refer to this person, or role, as 'the packager'. - -In terms of the \metadoc{}, the packager changes the description of the \executablething{} -from a \pythonprogram{} to a \dockercontainer{}. - -\begin{lstlisting}[] -executable: - type: uri:docker-container - repository: ghcr.io - image: ivoa/calycopis/java-builder - tag: 2024.08.30 - .... -\end{lstlisting} - -Depending on how the software is packaged in the container they may also need to update -the description of the inputs and outputs, -and link them to specific locations in the filesystem. - -\begin{lstlisting}[] -executable: - type: uri:docker-container - .... - parameters: - - type: uri:data-file - name: "input data" - format: - - type: urn:ivoa-votable - filename: input-data.vot - .... - -resources: - compute: - - type: uri:generic-compute - volumes: - - type: uri:file-mount - parameter: "input data" - filepath: /data - mode: readonly - .... -\end{lstlisting} - -\subsubsection{The client} -\label{user-metadata} - -The user, or the user's client agent, starts with the initial \metadoc{} from the -software discovery service, and adds additional information describing how the user -wants to use the software. - -Adding details of the data resources the user wants to use enables the \execbrokerservice{} -to transfer the data to local storage before the \execsession{} is started. - -Including a value for the filesize enables the \execbrokerservice{} to estimate -how much local storage it will need to allocate -and how much time will be needed to transfer the data. -The \execbrokerservice{} takes this into account when calculating the start time of -the \execoffer{s} it makes, allowing enough time for the data transfers to complete -before the \execsession{} starts. - -\begin{lstlisting}[] -resources: - data: - - type: uri:simple-data-resource - name: "input data" - location: http:data.example.org/.... - filesize: - value: 145 - units: MiB - .... -\end{lstlisting} - -Linking the data resources with volumes on the corresponding compute resources enables -the \execbrokerservice{} to mount the data resources at the correct location in -the compute resource's filesystem. - -\begin{lstlisting}[] -resources: - data: - - type: uri:simple-data-resource - name: input-data - .... - compute: - - type: uri:generic-compute - .... - volumes: - - resource: input-data - path: /data - mode: "ro" - -\end{lstlisting} - -The user can also update the compute resource requirements set by the software developer -to provide additional details based on how they plan to use the software. - -In the case of our square root example, the compute resource requirements set by the -developers will reflect the original intent of simply demonstrating how to use the -\pythonprogram{}. - -However, if the user intends to use the software to analyse a much larger \dataset{} -they can update the compute resource requirements to match their use case. - -\begin{lstlisting}[] -resources: - compute: - - type: uri:generic-compute - cores: - requested: - # min: 4 - min: 32 - memory: - requested: - # min: 16 - min: 128 - units: GiB -\end{lstlisting} - -TODO user provides the schedule ... when they want to run it. - \subsection{Execution broker} \label{execution-broker-intro} @@ -799,7 +593,7 @@ \subsubsection{Update options} Which options are available will depend on the current \codeword{state} of the \execsession{} and the identity and permissions of the authenticated user. -If the \execoffer{} is still being offered, then the list of available options +If the \execsession{} is still being offered, then the list of available options allow the user to accept or reject the offer by updating the \codeword{state} of the \execsession{} to \codeword{ACCEPTED} or \codeword{REJECTED}. @@ -836,8 +630,8 @@ \subsubsection{Update options} - "CANCELLED" \end{lstlisting} -This pattern of using a dynamic list of options enables an \execbrokerservice{} -communicate to the client what operations the user is able to do +Using a dynamic list of options in this way enables the \execbrokerservice{} +to communicate to the client what actions the user is able to take over the lifetime of an \execsession{}. \subsection{Session lifecycle} @@ -847,7 +641,6 @@ \subsection{Session lifecycle} \begin{itemize} - \item \codeword{PROPOSED} An internal state .. not published \item \codeword{OFFERED} The \workerjob{} is being offered. \item \codeword{ACCEPTED} The \workerjob{} has been accepted. \item \codeword{REJECTED} The \workerjob{} offer has been rejected. @@ -858,8 +651,8 @@ \subsection{Session lifecycle} \item \codeword{RUNNING} The \workerjob{} is executing. \item \codeword{RELEASING} The resources are being released. \item \codeword{COMPLETED} The \workerjob{} has completed. - \item \codeword{FAILED} The \workerjob{} has failed. \item \codeword{CANCELLED} The \workerjob{} has been cancelled. + \item \codeword{FAILED} The \workerjob{} has failed. \end{itemize} @@ -904,7 +697,238 @@ \subsection{Session lifecycle} completed, but they also need the \teardown{} data transfers to complete so that the results from this step are in the right place for the next step to be able to access them. -\section{The \executable{}} +\section{The data model} +\label{data-model} + +\subsection{Data curation roles} +\label{metadata-roles} + +The full description of an \executablething{} will include several layers of metadata +provided by different actors playing different roles within the publishing process. + +For our square root example we can identify a number of roles that would each provide +layers of the picture nedeed to fully describe an \executablething{}. + +The players: +\begin{itemize} + \item The developer - The person who wrote the \pythonprogram{} + \item The packager - The person who packaged it in a \dockercontainer{} + \item The publisher - The person who published it in a discovery service + \item The user agent - The person who wants to use the software +\end{itemize} + +\subsubsection{The developer} +\label{software-developer} + +The first layer of metadata comes from the person who wrote the \pythonprogram{}. +They have detailed knowledge of what the software does, what execution environment it needs, +and what the inputs and outputs are. + +For the square root example, it is a \pythonprogram{} which needs a platform with the \python{} runtime installed, +and a list of the \python{} libraries that the program relies on. + +\begin{lstlisting}[] +executable: + type: uri:python-program + requirements: + - numpi: "" + - astropy: ">= 6.1" +\end{lstlisting} + +The developer also understands how much memory their program needs, whether it can make use of multiple cpu cores, +and whether it can make use of a \gpu accelerator. + +\begin{lstlisting}[] +resources: + compute: + - type: uri:generic-compute + cores: + requested: + min: 4 + memory: + requested: + min: 16 + units: GiB + .... +\end{lstlisting} + +The developers also know about what inputs and outputs the program expects and what file +formats can it can handle. + +\begin{lstlisting}[] +executable: + type: uri:python-program + .... + parameters: + - type: uri:param-file + name: "input data" + mode: readonly + description: + A table containing a list of numbers to be processed, formatted as + comma separated text (CSV) or an IVOA VOTable. + formats: + - type: uri:text-csv + .... + - type: uri:votable + .... + - type: uri:param-value + name: "input column name" + type: string + description: + The column name within the 'input data' to use. +\end{lstlisting} + +\subsubsection{The packager} +\label{software-packager} + +Although it is possible to publish our square root example as a stand alone \pythonprogram{}, +it is not easy to describe the installation process in sufficient detail for it to be +automatically deployed on a range of different platforms. + +A more portable solution would be to package the \pythonprogram{} in a \dockercontainer{}, +installing and configuring the software along with all of its dependencies inside the container. + +This is often done as part of the software development process, but it is a separate +step that could be implemented by a different person. +To make this distinction clear we can refer to this person, or role, as 'the packager'. + +In terms of the \metadoc{}, the packager changes the description of the \executablething{} +from a \pythonprogram{} to a \dockercontainer{}. + +\begin{lstlisting}[] +executable: + type: uri:docker-container + repository: ghcr.io + image: ivoa/calycopis/java-builder + tag: 2024.08.30 + .... +\end{lstlisting} + +Depending on how the software is packaged in the container they may also need to update +the description of the inputs and outputs, +and link them to specific locations in the filesystem. + +\begin{lstlisting}[] +executable: + type: uri:docker-container + .... + parameters: + - type: uri:data-file + name: "input data" + format: + - type: urn:ivoa-votable + filename: input-data.vot + .... + +resources: + compute: + - type: uri:generic-compute + volumes: + - type: uri:file-mount + parameter: "input data" + filepath: /data + mode: readonly + .... +\end{lstlisting} + +\subsubsection{The publisher} +\label{metadata-publisher} + +This role represents the person who publishes metadata about the software in a discovery service. + +Typically changes made at this level would include adding to the description +of what the software does, placing it in a particular context to aid discovery, +or modifying the execution environment to configure the softweare for a particular domain. + +For example, this might include: + +\begin{itemize} + \item A project specific discovery service that only includes software vetted by the project. + Execution platforms within the project would only accept curated \metadoc{s} + from that discovery service. + \item A domain specific discovery service that modifies the execution environment, optimising + the software for analysing a particular type of data. + \item A catalog of \metadoc{s} maintained as part of a university teaching course, modifying the + execution environment to integrate the software into the university system and setting + parameters to configure the software to match the course notes. +\end{itemize} + +\subsubsection{The user} +\label{software-user} + +The user, or the user's client agent, starts with an initial \metadoc{} from the +software discovery service and adds additional information describing how the user +wants to use the software. + +Adding details of the data resources the user wants to use enables the \execbrokerservice{} +to transfer the data to local storage before the \execsession{} is started. + +Including a value for the filesize enables the \execbrokerservice{} to estimate +how much local storage it will need to allocate +and how much time will be needed to transfer the data. +The \execbrokerservice{} can take this into account when calculating the start time of +the \execoffer{s} it makes, allowing enough time for the data transfers to complete +before the \execsession{} starts. + +\begin{lstlisting}[] +resources: + data: + - type: uri:simple-data-resource + name: "input data" + location: http:data.example.org/.... + filesize: + value: 145 + units: MiB + .... +\end{lstlisting} + +Linking the data resources with volumes on the corresponding compute resources enables +the \execbrokerservice{} to mount the data resources at the correct location in +the compute resource's filesystem. + +\begin{lstlisting}[] +resources: + data: + - type: uri:simple-data-resource + name: input-data + .... + compute: + - type: uri:generic-compute + .... + volumes: + - resource: input-data + path: /data + mode: "ro" + +\end{lstlisting} + +The user can also update the compute resource requirements to reflect how they plan +to use the software. +In the case of our square root example, the compute resource requirements set by the +developers will reflect the original intent of simply demonstrating how to use the +\pythonprogram{}. + +However, if the user intends to use the software to analyse a much larger \dataset{} +they can update the compute resource requirements to match their use case. + +\begin{lstlisting}[] +resources: + compute: + - type: uri:generic-compute + cores: + requested: + # min: 4 + min: 32 + memory: + requested: + # min: 16 + min: 128 + units: GiB +\end{lstlisting} + +TODO user provides the schedule ... when they want to run it. + +\subsection{The \executable{}} \label{executable} At the simplest level the client just needs to check whether a platform is able to execute a particular @@ -940,7 +964,7 @@ \section{The \executable{}} spec: {} \end{lstlisting} -\subsection{\jupyternotebook{}} +\subsubsection{\jupyternotebook{}} \label{jupyternotebook} The \datamodel{} for each type of \executable{} defines the metadata needed to describe that particular type. @@ -979,7 +1003,7 @@ \subsection{\jupyternotebook{}} requirements: "https://.../requirements.txt" \end{lstlisting} -\subsection{\dockercontainer{}} +\subsubsection{\dockercontainer{}} \label{dockercontainer} The \datamodel{} for an \dockercontainer{} describes the image name and version along with the hostname of the repository to fetch it from. @@ -1021,7 +1045,7 @@ \subsection{\dockercontainer{}} Response - NO \end{lstlisting} -\section{Resources} +\subsection{Resources} \label{resources} At the next level the client may need to check whether a platform has sufficient compute resources @@ -1032,7 +1056,7 @@ \section{Resources} but also the minimum level of compute resources needed in terms of \cpu{} cores, memory, \gpu{s} and disc space needed to execute it. -\subsection{Compute resources} +\subsubsection{Compute resources} \label{compute-resources} The \datamodel{} for describing compute resources is, in most cases, common to all types of \executable{}, @@ -1215,119 +1239,7 @@ \subsection{Compute resources} \codeword{[https://github.com/..../resource-types/xilinx-vu19p.md]} \end{itemize} -\subsection{Minimum and maximum} -\label{minandmax} - -The \datamodel{} for describing compute resources includes elements for specifying the numeric size -and number of resources such as \cpu{} cores, memory and storage. - -If the \jupyternotebook{} in our example needs a minimum of 8 \cpu{} cores and 16G of memory -to be able to perform its calculations, then this can be specified in the compute resource. - -\begin{lstlisting}[] -# ExecutionBroker client request. -request: - # Details of the executable. - executable: - .... - # Details of the resources needed. - resources: - compute: - - name: "compute-001" - type: "https://www.purl.org/ivoa.net/resource-types/generic-compute" - spec: - cores: - min: 8 - memory: - min: 16 -\end{lstlisting} - -All of the \datamodel{} elements for specifying the size or number of resources are defined -as pairs of minimum and maximum values. -This allows a conversation between the \execbrokerclass{} client and services -to discover the best platform to execute the task. - -The client requests the minimun resources it needs, -and each service responds with a set of \execoffer{s} which specify the maximum -level of resources they are able to provide. - -For example, if a platform is able to provide double the compute resources, -16 \cpu{} cores and 32G of memory, -then it can indicate this by specifying higher maximum values in its response. - -\begin{lstlisting}[] -# ExecutionBroker service response. -response: - # Details of the executable. - executable: - .... - # Details of the resources offered. - resources: - compute: - - name: "compute-001" - type: "https://www.purl.org/ivoa.net/resource-types/generic-compute" - spec: - cores: - min: 8 - max: 16 - memory: - min: 16 - max: 32 -\end{lstlisting} - -This response represents an \execoffer{} to start with a minimum of 8 \cpu{} cores and 16G of memory -as requested, with the option to use a maximum of 16 \cpu{} cores and 32G of memory if needed. - -The client may receive different \execoffer{s} from different platforms and can pass this information -on the user to allow them to choose the \execoffer{} that best fits their use case. -The notebook they have chose may specify a minimum of 8 \cpu{} cores and 16G of memory, -but an \execoffer{} of twice the resources allows the user more scope for experimenting with -more data or more complex algorithms. - -This \scalable{} compute resource represents something like a \kubernetes{} platform where the -execution can start with a minimum configuration and scale on demand up to a maximum limit. - -This is slightly different to a platform like \openstack{} which allocates resources -in specific blocks, defined by the set of \textit{'flavors'} available on that particular platform. -If the smallest flavor of virtual machine available on the platform has 16 \cpu{} cores and 24G of memory, -then the service can represent that by setting the minimum values in its \execoffer{} to match available resources. - -\begin{lstlisting}[] -# ExecutionBroker service response. -response: - # Details of the executable. - executable: - .... - # Details of the resources offered. - resources: - compute: - - name: "compute-001" - type: "https://www.purl.org/ivoa.net/resource-types/generic-compute" - spec: - cores: - min: 16 - max: 16 - memory: - min: 24 - max: 24 -\end{lstlisting} - -This response represents an \execoffer{} to start with a fixed allocation of 16 \cpu{} cores and 24G of memory. - -An \execbrokerclass{} MAY NOT make an \execoffer{} with less than the minimum resources requested. -For example, if an \openstack{} platform only has a virtual machine flavor with 1 \cpu{} core and 2G of memory, -then it MAY NOT offer this resource as it is less than the requested minimum. - -Note that the term \textit{'compute resource'} specifically avoids stating whether the -notebook will be run in an \openstack{} virtual machine or a \kubernetes{} cluster. -As far as the user is concerned, it should not matter. As long as the compute resource provides -sufficient \cpu{} cores and memory for the notebook to execute, the technical details of how the -platform implements this should not be relevant at this level of abstraction. - -The user just wants their notebook to run. The technical details of what technologies are used -under the hood to make it happen should be someone else's problem. - -\subsection{Storage resources} +\subsubsection{Storage resources} \label{storage-resources} The resources section of the request can also be used to specify storage resources. @@ -1631,20 +1543,7 @@ \subsection{Storage resources} managing the lifecycle of the resources and releasing them automatically after they are no longer needed. -\section{Authentication} -\label{authentication} - -The client / service interactions requires some level of authentication to enable the -\execbrokerservice{} to control access to the - -The \executionbroker{} standard does not mandate a particular authentication method. -However, to facilitate interoperability with as wide a community as possible an individual \execbrokerclass{} -service should follow the guidlines set out in the \ivoa{} SSO standard. - - - - -\section{Date and time} +\subsection{Date and time} \label{date-time} The \codeword{datetime} part of the \datamodel{} enables the client and server to have a @@ -1814,7 +1713,6 @@ \section{Date and time} Technically the \datamodel{} allows an array of values for the \codeword{datetime} section, but this would impose unnecessary complexity on the client for no real gain in user experience. - \pagebreak \section{Federated architecture} diff --git a/notes/zrq/20241029-01-min-max.txt b/notes/zrq/20241029-01-min-max.txt new file mode 100644 index 0000000..a7f4d50 --- /dev/null +++ b/notes/zrq/20241029-01-min-max.txt @@ -0,0 +1,153 @@ +# +# +# +# Copyright (c) 2024, Manchester (http://www.manchester.ac.uk/) +# +# This information is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This information is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +# +# +#zrq-notes-indent +# +# AIMetrics: [] +# + + Target: + + Original section on min and max values. + Removed from the document, but needs to be replaced with a new section. + Saved the old text here for reference. + + Result: + + Work in progress ... + +# ----------------------------------------------------- + +subsection{Minimum and maximum} +\label{minandmax} + +The \datamodel{} for describing compute resources includes elements for specifying the numeric size +and number of resources such as \cpu{} cores, memory and storage. + +If the \jupyternotebook{} in our example needs a minimum of 8 \cpu{} cores and 16G of memory +to be able to perform its calculations, then this can be specified in the compute resource. + +\begin{lstlisting}[] +# ExecutionBroker client request. +request: + # Details of the executable. + executable: + .... + # Details of the resources needed. + resources: + compute: + - name: "compute-001" + type: "https://www.purl.org/ivoa.net/resource-types/generic-compute" + spec: + cores: + min: 8 + memory: + min: 16 +\end{lstlisting} + +All of the \datamodel{} elements for specifying the size or number of resources are defined +as pairs of minimum and maximum values. +This allows a conversation between the \execbrokerclass{} client and services +to discover the best platform to execute the task. + +The client requests the minimun resources it needs, +and each service responds with a set of \execoffer{s} which specify the maximum +level of resources they are able to provide. + +For example, if a platform is able to provide double the compute resources, +16 \cpu{} cores and 32G of memory, +then it can indicate this by specifying higher maximum values in its response. + +\begin{lstlisting}[] +# ExecutionBroker service response. +response: + # Details of the executable. + executable: + .... + # Details of the resources offered. + resources: + compute: + - name: "compute-001" + type: "https://www.purl.org/ivoa.net/resource-types/generic-compute" + spec: + cores: + min: 8 + max: 16 + memory: + min: 16 + max: 32 +\end{lstlisting} + +This response represents an \execoffer{} to start with a minimum of 8 \cpu{} cores and 16G of memory +as requested, with the option to use a maximum of 16 \cpu{} cores and 32G of memory if needed. + +The client may receive different \execoffer{s} from different platforms and can pass this information +on the user to allow them to choose the \execoffer{} that best fits their use case. +The notebook they have chose may specify a minimum of 8 \cpu{} cores and 16G of memory, +but an \execoffer{} of twice the resources allows the user more scope for experimenting with +more data or more complex algorithms. + +This \scalable{} compute resource represents something like a \kubernetes{} platform where the +execution can start with a minimum configuration and scale on demand up to a maximum limit. + +This is slightly different to a platform like \openstack{} which allocates resources +in specific blocks, defined by the set of \textit{'flavors'} available on that particular platform. +If the smallest flavor of virtual machine available on the platform has 16 \cpu{} cores and 24G of memory, +then the service can represent that by setting the minimum values in its \execoffer{} to match available resources. + +\begin{lstlisting}[] +# ExecutionBroker service response. +response: + # Details of the executable. + executable: + .... + # Details of the resources offered. + resources: + compute: + - name: "compute-001" + type: "https://www.purl.org/ivoa.net/resource-types/generic-compute" + spec: + cores: + min: 16 + max: 16 + memory: + min: 24 + max: 24 +\end{lstlisting} + +This response represents an \execoffer{} to start with a fixed allocation of 16 \cpu{} cores and 24G of memory. + +An \execbrokerclass{} MAY NOT make an \execoffer{} with less than the minimum resources requested. +For example, if an \openstack{} platform only has a virtual machine flavor with 1 \cpu{} core and 2G of memory, +then it MAY NOT offer this resource as it is less than the requested minimum. + +Note that the term \textit{'compute resource'} specifically avoids stating whether the +notebook will be run in an \openstack{} virtual machine or a \kubernetes{} cluster. +As far as the user is concerned, it should not matter. As long as the compute resource provides +sufficient \cpu{} cores and memory for the notebook to execute, the technical details of how the +platform implements this should not be relevant at this level of abstraction. + +The user just wants their notebook to run. The technical details of what technologies are used +under the hood to make it happen should be someone else's problem. + + + + +