From 7ab1604d6c798f40fd2fff2a670aed59d5d1159d Mon Sep 17 00:00:00 2001 From: Zarquan Date: Fri, 29 Nov 2024 05:32:11 +0000 Subject: [PATCH 1/2] format-notes --- notes/zrq/20230807-01-git-init.txt | 66 +++++++++++++----------------- 1 file changed, 29 insertions(+), 37 deletions(-) diff --git a/notes/zrq/20230807-01-git-init.txt b/notes/zrq/20230807-01-git-init.txt index dc7f8bd..28ed75b 100644 --- a/notes/zrq/20230807-01-git-init.txt +++ b/notes/zrq/20230807-01-git-init.txt @@ -64,43 +64,35 @@ git remote add origin 'git@github.com:Zarquan/ExecPlannerNote2.git' git push --set-upstream origin main ---START-- -Initialized empty Git repository in /var/local/projects/IVOA/EP-TDL/ExecPlannerNote2/github-zrq/.git/ ---END-- - ---START-- -Cloning into '/var/local/projects/IVOA/EP-TDL/ExecPlannerNote2/github-zrq/ivoatex'... -remote: Enumerating objects: 1422, done. -remote: Counting objects: 100% (171/171), done. -remote: Compressing objects: 100% (42/42), done. -remote: Total 1422 (delta 134), reused 137 (delta 129), pack-reused 1251 -Receiving objects: 100% (1422/1422), 2.15 MiB | 6.73 MiB/s, done. -Resolving deltas: 100% (920/920), done. ---END-- - ---START-- -[main (root-commit) 0195ac2] Creating initial templates for ExecPlannerNote2 - 7 files changed, 588 insertions(+) - create mode 100644 .gitignore - create mode 100644 .gitmodules - create mode 100644 ExecPlannerNote2.tex - create mode 100644 LICENSE - create mode 100644 Makefile - create mode 100644 README.md - create mode 160000 ivoatex ---END-- - ---START-- -Enumerating objects: 8, done. -Counting objects: 100% (8/8), done. -Delta compression using up to 4 threads -Compressing objects: 100% (8/8), done. -Writing objects: 100% (8/8), 8.95 KiB | 2.24 MiB/s, done. -Total 8 (delta 0), reused 0 (delta 0), pack-reused 0 -To github.com:Zarquan/ExecPlannerNote2.git - * [new branch] main -> main -branch 'main' set up to track 'origin/main'. ---END-- + > Initialized empty Git repository in /var/local/projects/IVOA/EP-TDL/ExecPlannerNote2/github-zrq/.git/ + + > Cloning into '/var/local/projects/IVOA/EP-TDL/ExecPlannerNote2/github-zrq/ivoatex'... + > remote: Enumerating objects: 1422, done. + > remote: Counting objects: 100% (171/171), done. + > remote: Compressing objects: 100% (42/42), done. + > remote: Total 1422 (delta 134), reused 137 (delta 129), pack-reused 1251 + > Receiving objects: 100% (1422/1422), 2.15 MiB | 6.73 MiB/s, done. + > Resolving deltas: 100% (920/920), done. + + > [main (root-commit) 0195ac2] Creating initial templates for ExecPlannerNote2 + > 7 files changed, 588 insertions(+) + > create mode 100644 .gitignore + > create mode 100644 .gitmodules + > create mode 100644 ExecPlannerNote2.tex + > create mode 100644 LICENSE + > create mode 100644 Makefile + > create mode 100644 README.md + > create mode 160000 ivoatex + + > Enumerating objects: 8, done. + > Counting objects: 100% (8/8), done. + > Delta compression using up to 4 threads + > Compressing objects: 100% (8/8), done. + > Writing objects: 100% (8/8), 8.95 KiB | 2.24 MiB/s, done. + > Total 8 (delta 0), reused 0 (delta 0), pack-reused 0 + > To github.com:Zarquan/ExecPlannerNote2.git + > * [new branch] main -> main + > branch 'main' set up to track 'origin/main'. From 35afebce98c1743f81ff3ef6afd57e70bdfc0c96 Mon Sep 17 00:00:00 2001 From: Zarquan Date: Fri, 29 Nov 2024 05:32:20 +0000 Subject: [PATCH 2/2] Gardening --- ExecutionBroker.tex | 451 ++++++++++++++++++---------- notes/zrq/20241128-01-gardening.txt | 200 ++++++++++++ 2 files changed, 492 insertions(+), 159 deletions(-) create mode 100644 notes/zrq/20241128-01-gardening.txt diff --git a/ExecutionBroker.tex b/ExecutionBroker.tex index e8a6dc5..db09a2e 100644 --- a/ExecutionBroker.tex +++ b/ExecutionBroker.tex @@ -32,6 +32,8 @@ \newcommand{\execworkerclass} {**ExecutionWorker**} \newcommand{\execbrokerclass} {\textit{ExecutionBroker}} \newcommand{\execbrokerservice}[1] {\textit{ExecutionBroker~service#1}} +\newcommand{\execbrokersession}[1] {\textit{ExecutionBroker~session#1}} +\newcommand{\execbrokerstate}[1] {\codeword{state#1}} \newcommand{\execoffer}[1] {\textit{ExecutionBroker~offer#1}} \newcommand{\execofferset}[1] {\textit{ExecutionBroker~offerset#1}} @@ -39,6 +41,7 @@ \newcommand{\executionbroker} {\textit{Execution~Broker}} \newcommand{\executionplanning} {\textit{Execution~Planning}} +\newcommand{\execplatform} {\textit{execution~platform}} \newcommand{\executable} {\textit{executable}} \newcommand{\executablething}[1] {\textit{executable~thing#1}} @@ -170,7 +173,7 @@ % https://mirror.ox.ac.uk/sites/ctan.org/macros/latex/contrib/listings/listings.pdf#subsection.3.3 \renewcommand{\ttdefault}{pcr} \lstset{moredelim=[is][\bfseries]{[*}{*]}} - +\lstset{moredelim=[is][\itshape]{[+}{+]}} \title{IVOA Execution Broker} @@ -221,12 +224,13 @@ \end{abstract} \section*{Acknowledgments} -\label{acknowledgments} +\label{sect-acknowledgments} The authors would like to thank all the participants in the IVOA and ESCAPE projects who have contributed their ideas, critical reviews, and suggestions to this document. \section*{Conformance-related definitions} +\label{sect-conformance} The words ``MUST'', ``SHALL'', ``SHOULD'', ``MAY'', ``RECOMMENDED'', and ``OPTIONAL'' (in upper or lower case) used in this document are to be @@ -240,7 +244,7 @@ \section*{Conformance-related definitions} infrastructure that enable VO applications. \section{Introduction} -\label{introduction} +\label{sect-introduction} The \ivoa{} \executionbroker{} specification defines a \datamodel{} for describing executable tasks and a \webservice{} interface for managing them. @@ -254,7 +258,7 @@ \section{Introduction} \end{itemize} \subsection{Role within the VO Architecture} -\label{subsec:ivoarole} +\label{sub-ivoa-role} % As of ivoatex 1.2, the architecture diagram is generated by ivoatex in % SVG; copy ivoatex/archdiag-full.xml to role_diagram.xml and throw out @@ -295,10 +299,10 @@ \subsection{Role within the VO Architecture} of execution platforms. \subsection{Supplementary documents} -\label{supplementary-documents} +\label{sub-supplementary-documents} \subsubsection{\openapi{} specification} -\label{openapi-specification} +\label{subsub-openapi-specification} This document is designed to read in combination with the \openapi{} \footurl{https://www.openapis.org/} \footurl{https://swagger.io/specification/} @@ -332,7 +336,7 @@ \subsubsection{\openapi{} specification} definitive source and this text document should be considered as secondary. \subsubsection{IVOA profiles} -\label{ivoa-profiles} +\label{subsub-ivoa-profiles} This specification refers to the following documents to cover the details of how an \executionbroker{} \webservice{} @@ -362,16 +366,16 @@ \subsubsection{IVOA profiles} should follow the guidelines outined in these documents. \subsubsection{IVOA Single-Sign-On} -\label{ivoa-sso} +\label{subsub-ivoa-sso} -In addition, an \ivoa{} \executionbroker{} service may use parts of the +An \ivoa{} \executionbroker{} service MAY use parts of the \ivoa{} Single-Sign-On standard\citep{2017ivoa.spec.0524T} for authentication, and the \ivoa{} Credential Delegation Protocol \citep{2010ivoa.spec.0218P} for delegating credentials to other services. \subsection{Executable things} -\label{executablething} +\label{sub-executablething} To understand the problem that the \ivoa{} \executionbroker{} is trying to solve it is useful to describe what an \executablething{} is in this context. @@ -437,14 +441,14 @@ \subsection{Executable things} the calculation, and the storage space needed to save the results. \section{Service interaction} -\label{service-interaction} +\label{sect-service-interaction} -The interaction between a user, the client application they are using, and the services available in the \vo{} -can be described as a conversation between the client and one or more \execbrokerservice{s} to discover -where, how, and when, an \executablething{} can be executed. +The interaction between a user, the client application they are using, and the services available in the \vofull{} +can be described as a conversation to discover where, how, and when, an \executablething{} that the user +has chosen can be executed. \subsection{Discovery services} -\label{discovery-services} +\label{sub-discovery-services} The conversation starts at the discovery stage, where the user uses discovery services to select the software and \dataset{s} that they want to work with. @@ -477,7 +481,7 @@ \subsection{Discovery services} should be the same. \subsubsection{Software discovery} -\label{software-discovery} +\label{subsub-software-discovery} There are three main components involved in software discovery, the metadata schema for describing the software, one or more search services, and the @@ -553,29 +557,27 @@ \subsubsection{Software discovery} \end{itemize} The other components in the software discovery stack, the database of search terms, and -storage and access services for the \metadoc{s} and binary images, do not need to be +the storage and access services for the \metadoc{s} and binary images, do not need to be standardised at this stage. \subsubsection{Data discovery} -\label{data-discovery} +\label{subsub-data-discovery} TODO : update this with reference to \ivoa{} data product type. https://www.ivoa.net/rdf/product-type/2024-05-19/product-type.html -\subsection{Execution broker} -\label{execution-broker-intro} - -\subsubsection{OfferSet request} -\label{offerset-request} +\subsection{Execution Broker} +\label{sub-execution-broker} -Once the user has selected the \executablething{} they want to use and the -data they want to apply it to, the client combines this information to create a -complete description of the \execsession{} the user wants to execute, including -details of the executable, the compute, storage, and data resources it needs, -and a schedule describing when the user wants it to run. +Once the user has identified the software and data resources that they want to use, +the client application brings together details of the \executablething{} the user +has selected, the compute, storage, and data resources they want to use, +along with a schedule describing when the user wants it to execute, +to create a \metadoc{} description of the \execsession{} the user wants +to execute. \begin{lstlisting}[] -# ExecutionBroker OfferSet request. +[+# ExecutionBroker OfferSet request.+] executable: .... resources: @@ -584,6 +586,9 @@ \subsubsection{OfferSet request} .... \end{lstlisting} +\subsubsection{OfferSet request} +\label{subsub-offerset-request} + The client sends the \metadoc{} description to one or more \execbrokerclass{} services to ask if they can meet the requirements and execute the \execsession{}. @@ -594,17 +599,108 @@ \subsubsection{OfferSet request} that \execbrokerservice{}. %\begin{lstlisting}[] -%Request - Can this platform execute ? +%[+# ExecutionBroker OfferSet request/response.+] +%Request - Can this platform execute ? %Response - YES, list of [] %\end{lstlisting} \includegraphics[width=0.9\textwidth]{diagrams/request-offers.pdf} + +\subsubsection{Resource allocation} +\label{subsub-resource-allocation} + +The \execbrokerservice{} is responsible for managing the resources +within the underlying \execplatform{} that it represents. + +The \execbrokerservice{} is responsible for keeping track of +reservations for the resources that it grants to clients in +response to execution requests. + +The details of how an \execbrokerservice{} implements this is +beyond the scope of this specification. + +This specification defines a set of black-box behaviours +that an \execbrokerservice{} MUST implement. + +\begin{itemize} + \item Each \execoffer{} MUST meet the minimum criteria specified in the request. + \item Each \execoffer{} MUST be within the capabilities of the underlying \execplatform{} to be able to provide. + + \item An \execbrokerservice{} MAY offer more resources than the minimum value specified in the request. + \item An \execbrokerservice{} MAY offer more execution time beyond the minimum value specified in the request. + + \item An \execbrokerservice{} SHOULD NOT offer more resources than the maximum value specified in the request. + \item An \execbrokerservice{} SHOULD NOT offer more execution time beyond the maximum value specified in the request. +\end{itemize} + +This specification recognises a set of black-box behaviours that an \execbrokerservice{} SHOULD implement. + +\begin{itemize} + \item An \execoffer{} made by an \execbrokerservice{} is assumed to be made in good faith, with a reasonable probability of success. + \item An \execbrokerservice{} is free to just say \codeword{NO} to a request, for whatever reason. +\end{itemize} + +In addition, this specification recognises a number of situational and dispositional factors +that may influence how an \execbrokerservice{} allocates resources. + +\begin{itemize} + \item Different implementations may represent a range of different types of \execplatform{s}. + \begin{itemize} + \item Science platforms. + \begin{itemize} + \item CANFAR. + \item Azimuth. + \item SciServer. + \end{itemize} + \item Cloud platforms. + \begin{itemize} + \item Kubernetes platforms. + \item Openstack platforms. + \item Commercial cloud platforms. + \end{itemize} + \item Service platforms. + \begin{itemize} + \item Slurm services. + \end{itemize} + \end{itemize} + + \item Different implementations may be connected to a range of different sizes of \execplatform{s}. + \begin{itemize} + \item Large scale HPC systems. + \item Large scale commercial platforms. + \item Medium scale on-premises platforms. + \item Small scale specialist platforms. + \end{itemize} + + \item Different deployments may apply different algorithms to manage the allocation of resources. + \begin{itemize} + \item Maximise the amount of resources offered to users. + \item Maximise the number of jobs a \execplatform{} can execute. + \item Prioritise interactive sessions by offering precise start times. + \item Prioritise batch processing by offering wide start ranges. + \end{itemize} + + \item Different deployments may apply different policies to manage the allocation of resources. + \begin{itemize} + \item Required security criteria. + \item Recognised identity providers. + \item Required group membership policies. + \item Local resource allocation policies. + \item Local white lists of allowed executables. + \end{itemize} +\end{itemize} + +\subsubsection{Execution scheduling} +\label{subsub-execution-scheduling} + + + \subsubsection{Offerset response} -\label{offerset-response} +\label{subsub-offerset-response} Each \execbrokerservice{} will respond to a request for offers with an \execofferset{} -containing some metadata about the \execofferset{} itself, and a list of \execoffer{}s +containing some metadata about the \execofferset{} itself, followed by a list of \execoffer{}s describing how the requested \execsession{} could be executed. Each \execoffer{} in the list contains some metadata about the \execoffer{} itself, @@ -612,13 +708,12 @@ \subsubsection{Offerset response} \execsession{} would be executed. \begin{lstlisting}[] -# ExecutionBroker OfferSet response. +[+# ExecutionBroker OfferSet response.+] result: YES .... offers: - uuid: "2e164a1b-7ff6-11ef-8412-4bc36fe2face" - href: "http://..../sessions/2e164a1b-7ff6-11ef-8412-4bc36fe2face" state: 'OFFERED' expires: "2023-09-18T07:05:21" .... @@ -630,7 +725,6 @@ \subsubsection{Offerset response} .... - uuid: "2e16bf4c-7ff6-11ef-8412-4bc36fe2face" - href: "http://..../sessions/2e16bf4c-7ff6-11ef-8412-4bc36fe2face" status: 'OFFERED' expires: "2023-09-18T07:05:21" .... @@ -642,89 +736,65 @@ \subsubsection{Offerset response} .... \end{lstlisting} -The user can choose to accept one of the \execoffer{s} from the list that best -fits their requirements, or they can reject the \execoffer{s} and make a new -request with different criteria. +\subsubsection{Session lifecycle} +\label{subsub-session-lifecycle} + +An \execbrokersession{} goes through the following lifecycle stages. + +\begin{itemize} + \item \codeword{OFFERED} The \workerjob{} is being offered. + \item \codeword{ACCEPTED} The \workerjob{} has been accepted. + \item \codeword{REJECTED} The \workerjob{} offer has been rejected. + \item \codeword{EXPIRED} The \workerjob{} offer has expired. + \item \codeword{WAITING} The \workerjob{} is waiting to activate. + \item \codeword{PREPARING} The resources are being prepared. + \item \codeword{READY} The \workerjob{} is ready to execute. + \item \codeword{RUNNING} The \workerjob{} is executing. + \item \codeword{RELEASING} The resources are being released. + \item \codeword{COMPLETED} The \workerjob{} has completed. + \item \codeword{CANCELLED} The \workerjob{} has been cancelled. + \item \codeword{FAILED} The \workerjob{} has failed. +\end{itemize} + +The first four \execbrokerstate{s} are interactive, where the \execbrokerstate{} changes +are primarily driven by user interactions. + +\begin{itemize} + \item \codeword{OFFERED} The \workerjob{} is being offered. + \item \codeword{ACCEPTED} The \workerjob{} has been accepted. + \item \codeword{REJECTED} The \workerjob{} offer has been rejected. + \item \codeword{EXPIRED} The \workerjob{} offer has expired. +\end{itemize} + +When the \execbrokerservice{} replies with an \execofferset{} containing +a list of \execoffer{}s, the user can choose to do nothing, to accept one of the +\execoffer{s} from the list that best +fits their requirements, or they can reject the \execoffer{s}. Each of the \execoffer{s} in the \execofferset{} represent a temporary reservation for the resources listed in the \execoffer{}. This means these resources will not available to other users while the \execoffer{s} are still valid. -If the user does nothing, then \codeword{state} of each of the \execoffer{s} will +If the user does nothing, then \execbrokerstate{} of each of the \execoffer{s} will automatically be updated to \codeword{EXPIRED}, and their associated resources will be released, when their expiry time is reached. If the user accepts one of the \execoffer{s} in the \execofferset{} by updating -the \codeword{state} to \codeword{ACCEPTED}, the \execbrokerservice{} SHOULD -update the \codeword{state} of the other \execoffer{s} in the \execofferset{} to +the \execbrokerstate{} to \codeword{ACCEPTED}, the \execbrokerservice{} SHOULD +update the \execbrokerstate{} of the other \execoffer{s} in the \execofferset{} to \codeword{REJECTED} and release the associated resources. TODO [accept/reject/expire state-transition diagram ] -\subsubsection{Update options} -\label{update-options} - -Each \execsession{} has a unique URL that the client can use to monitor and update -its state. - -The \execbrokerservice{} response for an \execsession{} includes a list of options -that the user may use to update or modify the \execsession{}. -Which options are available will depend on the current \codeword{state} of the -\execsession{} and the identity and permissions of the authenticated user. - -If the \execsession{} is still being offered, then the list of available options -allow the user to accept or reject the offer by updating the \codeword{state} of -the \execsession{} to \codeword{ACCEPTED} or \codeword{REJECTED}. - -\begin{lstlisting}[] -uuid: "2e164a1b-7ff6-11ef-8412-4bc36fe2face" -href: "http://..../sessions/2e164a1b-7ff6-11ef-8412-4bc36fe2face" -state: 'OFFERED' -expires: "2023-09-18T07:05:21" -.... -.... -options: - - type: "uri:enum-value-option" - path: "state" - values: - - "ACCEPTED" - - "REJECTED" -\end{lstlisting} - -Once the \execoffer{} has been accepted and the \execsession{} has started to -execute, then the list of available options will only allow the user to cancel -the execution. - -\begin{lstlisting}[] -uuid: "2e164a1b-7ff6-11ef-8412-4bc36fe2face" -href: "http://..../sessions/2e164a1b-7ff6-11ef-8412-4bc36fe2face" -state: 'ACCEPTED' -expires: "2023-09-18T07:05:21" -.... -.... -options: - - type: "uri:enum-value-option" - path: "state" - values: - - "CANCELLED" -\end{lstlisting} - -Using a dynamic list of options in this way enables the \execbrokerservice{} -to communicate to the client what actions the user is able to take -over the lifetime of an \execsession{}. - -\subsection{Session lifecycle} -\label{session-lifecycle} +\subsubsection{Session execution} +\label{subsub-session-execution} -A \workerjob{} in an \execworkerclass{} service goes through the following stages in its lifecycle. +Once the user accepts one of the \execoffer{s}, the \execbrokerservice{} takes over and +drives the subsequent \execbrokerstate{} changes based on the timing schedule and +the results of the internal steps within the \execbrokersession{}. \begin{itemize} - - \item \codeword{OFFERED} The \workerjob{} is being offered. - \item \codeword{ACCEPTED} The \workerjob{} has been accepted. - \item \codeword{REJECTED} The \workerjob{} offer has been rejected. - \item \codeword{EXPIRED} The \workerjob{} offer has expired. \item \codeword{WAITING} The \workerjob{} is waiting to activate. \item \codeword{PREPARING} The resources are being prepared. \item \codeword{READY} The \workerjob{} is ready to execute. @@ -733,14 +803,14 @@ \subsection{Session lifecycle} \item \codeword{COMPLETED} The \workerjob{} has completed. \item \codeword{CANCELLED} The \workerjob{} has been cancelled. \item \codeword{FAILED} The \workerjob{} has failed. - \end{itemize} -%ZRQ - urgh -When a \execbrokerclass{} creates a \workerjob{} in an \execworkerclass{} service the -\workerjob{} starts with the \codeword{phase} set to \codeword{PENDING}. -It is up to the \execworkerclass{} to select the right time to change the \workerjob{} + + + + +It is up to the \execbrokerservice{} to select the right time to change the \workerjob{} \codeword{phase} from \codeword{WAITING} to \codeword{PEPARING} and begin preparing the resources so that the \workerjob{} is \codeword{READY} in time for the \codeword{starttime} declared in the \execoffer{}. @@ -778,11 +848,71 @@ \subsection{Session lifecycle} completed, but they also need the \teardown{} data transfers to complete so that the results from this step are in the right place for the next step to be able to access them. + + + + + + + + +\subsubsection{Update options} +\label{subsub-update-options} + +Each \execsession{} has a unique URL that the client can use to monitor and update +its state. + +The \execbrokerservice{} response for an \execsession{} includes a list of options +that the user may use to update or modify the \execsession{}. +Which options are available will depend on the current \codeword{state} of the +\execsession{} and the identity and permissions of the authenticated user. + +If the \execsession{} is still being offered, then the list of available options +allow the user to accept or reject the offer by updating the \codeword{state} of +the \execsession{} to \codeword{ACCEPTED} or \codeword{REJECTED}. + +\begin{lstlisting}[] +uuid: "2e164a1b-7ff6-11ef-8412-4bc36fe2face" +href: "http://..../sessions/2e164a1b-7ff6-11ef-8412-4bc36fe2face" +state: 'OFFERED' +expires: "2023-09-18T07:05:21" +.... +.... +options: + - type: "uri:enum-value-option" + path: "state" + values: + - "ACCEPTED" + - "REJECTED" +\end{lstlisting} + +Once the \execoffer{} has been accepted and the \execsession{} has started to +execute, then the list of available options will only allow the user to cancel +the execution. + +\begin{lstlisting}[] +uuid: "2e164a1b-7ff6-11ef-8412-4bc36fe2face" +href: "http://..../sessions/2e164a1b-7ff6-11ef-8412-4bc36fe2face" +state: 'ACCEPTED' +expires: "2023-09-18T07:05:21" +.... +.... +options: + - type: "uri:enum-value-option" + path: "state" + values: + - "CANCELLED" +\end{lstlisting} + +Using a dynamic list of options in this way enables the \execbrokerservice{} +to communicate to the client what actions the user is able to take +over the lifetime of an \execsession{}. + \section{The data model} -\label{data-model} +\label{sect-data-model} \subsection{Metadata roles} -\label{metadata-roles} +\label{sub-metadata-roles} The full description of an \executablething{} will include several layers of metadata provided by different actors playing different roles within the publishing process. @@ -799,7 +929,7 @@ \subsection{Metadata roles} \end{itemize} \subsubsection{The developer} -\label{software-developer} +\label{subsub-software-developer} The first layer of metadata comes from the person who wrote the software. They have detailed knowledge of what the software does, what execution environment it needs, @@ -863,7 +993,7 @@ \subsubsection{The developer} \end{lstlisting} \subsubsection{The packager} -\label{software-packager} +\label{subsub-software-packager} Although it is possible to publish our square root example as a stand alone \pythonprogram{}, it is not easy to describe the installation process in sufficient detail for it to be @@ -885,6 +1015,7 @@ \subsubsection{The packager} to a \dockercontainer{}. \begin{lstlisting}[] +[+# Executable metadata+] executable: name: Newton-Raphson example [*type: uri:docker-container-1.0*] @@ -901,6 +1032,7 @@ \subsubsection{The packager} %https://github.com/ivoa-std/ExecutionBroker/issues/89 \begin{lstlisting}[] +[+# Executable metadata+] executable: name: Newton-Raphson example type: uri:docker-container-1.0 @@ -925,7 +1057,7 @@ \subsubsection{The packager} \end{lstlisting} \subsubsection{The publisher} -\label{metadata-publisher} +\label{subsub-metadata-publisher} This role represents the person who publishes metadata about the software in a discovery service. @@ -947,7 +1079,7 @@ \subsubsection{The publisher} \end{itemize} \subsubsection{The user} -\label{software-user} +\label{subsub-software-user} The user starts with an initial \metadoc{} from the software discovery service and adds additional information describing how they @@ -1036,7 +1168,7 @@ \subsubsection{The user} TODO user provides the schedule to describe when they want to run it. \subsection{The \executable{}} -\label{executable} +\label{sub-executable} At the simplest level the client just needs to check whether a platform is able to execute a particular type of \excutabletask{}. @@ -1135,7 +1267,7 @@ \subsection{The \executable{}} \end{lstlisting} \subsubsection{\jupyternotebook{}} -\label{jupyternotebook} +\label{subsub-jupyternotebook} The \datamodel{} for each type of \executable{} defines the metadata needed to describe that particular type. For example, the \datamodel{} for a \jupyternotebook{} needs to describe where @@ -1174,7 +1306,7 @@ \subsubsection{\jupyternotebook{}} \end{lstlisting} \subsubsection{\dockercontainer{}} -\label{dockercontainer} +\label{subsub-dockercontainer} The \datamodel{} for an \dockercontainer{} describes the image name and version along with the hostname of the repository to fetch it from. @@ -1216,7 +1348,7 @@ \subsubsection{\dockercontainer{}} \end{lstlisting} \subsection{Resources} -\label{resources} +\label{sub-resources} At the next level the client may need to check whether a platform has sufficient compute resources needed to execute a particular task. @@ -1227,7 +1359,7 @@ \subsection{Resources} and disc space needed to execute it. \subsubsection{Compute resources} -\label{compute-resources} +\label{subsub-compute-resources} The \datamodel{} for describing compute resources is, in most cases, common to all types of \executable{}, so the \datamodel{} for these requirements are defined as part of the core \executionbroker{} \datamodel{}. @@ -1410,7 +1542,7 @@ \subsubsection{Compute resources} \end{itemize} \subsubsection{Storage resources} -\label{storage-resources} +\label{subsub-storage-resources} The resources section of the request can also be used to specify storage resources. @@ -1714,7 +1846,7 @@ \subsubsection{Storage resources} are no longer needed. \subsection{Date and time} -\label{date-time} +\label{sub-date-time} The \codeword{datetime} part of the \datamodel{} enables the client and server to have a conversation about when a \workerjob{} can be executed. @@ -1886,7 +2018,7 @@ \subsection{Date and time} \pagebreak \section{Federated architecture} -\label{federation} +\label{sect-sect-federation} The \execbrokerservice{} is designed to be used at multiple levels within an organization. @@ -2009,7 +2141,7 @@ \section{Federated architecture} \end{itemize} \section{Example use cases} -\label{example-usecases} +\label{sect-example-usecases} This section contains a set of examples ranging from very simple to complex, with a complete listing of the \datamodel{} used to describe each case. @@ -2020,55 +2152,55 @@ \section{Example use cases} \codeword{NO} to everything else. \subsection{Simple notebook} -\label{simple-notebook} +\label{sub-simple-notebook} Just a simple \jupyternotebook{}. ... \subsection{Notebook with dates} -\label{notebook-with-dates} +\label{sub-notebook-with-dates} A \jupyter{} notebook with specific date ranges for start time. ... \subsection{Notebook with data} -\label{notebook-with-data} +\label{sub-notebook-with-data} A \jupyternotebook{} with specific input data. ... \subsection{Notebook with compute} -\label{notebook-with-compute} +\label{sub-notebook-with-compute} A \jupyternotebook{} with specific input data and compute resources. ... \subsection{Simple container} -\label{simple-container} +\label{sub-simple-container} A simple \dockercontainer{}. ... \subsection{Container with data} -\label{container-with-data} +\label{sub-container-with-data} An \dockercontainer{} with specific input data. ... \subsection{Container with compute} -\label{container-with-compute} +\label{sub-container-with-compute} An \dockercontainer{} with specific input and output data, and compute resources. ... \subsection{Kubernetes Helm chart} -\label{kubernetes-helm} +\label{sub-kubernetes-helm} A complex Helm chart with multiple elements launched as a single task. ... \subsection{Spark cluster} -\label{spark-cluster} +\label{sub-spark-cluster} A complex Spark cluster analysis launched as a single task. ... @@ -2076,13 +2208,13 @@ \subsection{Spark cluster} \pagebreak \section{Service specification} -\label{service-specification} +\label{sect-service-specification} Details of the request and response messages. Timeline of request, offer and accept. \subsection{General requirements} -\label{general-requirements} +\label{sub-general-requirements} \begin{itemize} \item .... @@ -2091,14 +2223,14 @@ \subsection{General requirements} \end{itemize} \subsection{ExecutionBroker} -\label{execution-planner-spec} +\label{sub-execution-planner-spec} The main \execbrokerclass{} \webservice{} implements two interfaces; a \codeword{POST} method for sending requests, and a \codeword{GET/POST} interface for accessing \execoffer{s} and updating their status. \subsubsection{Request method} -\label{execution-planner-request} +\label{subsub-execution-planner-request} The main \execbrokerclass{} \webservice{} method is a \codeword{POST} \webservice{} that accepts a \codeword{request} as defined in section \ref{datamodel-request} of the \datamodel{}. @@ -2199,7 +2331,7 @@ \subsubsection{Request method} \end{itemize} \subsubsection{Offer GET} -\label{execution-planner-offer-get} +\label{subsub-execution-planner-offer-get} The \codeword{offer} \codeword{GET} method requests the current state of an \execoffer{}. The identifier for the \execoffer{} is encoded in the URL of the HTTP request by appending the @@ -2244,7 +2376,7 @@ \subsubsection{Offer GET} \end{lstlisting} \subsubsection{Offer POST} -\label{execution-planner-offer-post} +\label{subsub-execution-planner-offer-post} The \codeword{offer} \codeword{POST} method provides the client with a method for updating the status of an \codeword{offer}. @@ -2312,7 +2444,7 @@ \subsubsection{Offer POST} \pagebreak \section{The \datamodel{}} -\label{datamodel} +\label{sect-datamodel} The following section describes the core \datamodel{} using a YAML syntax to define the structure with comments to describe the elements. @@ -2322,7 +2454,7 @@ \section{The \datamodel{}} into a number of different formats, including YAML, JSON and XML. \subsection{Type URIs and specifications} -\label{type-and-spec} +\label{sub-type-and-spec} Several parts of the \ivoa{} \executionbroker{} \datamodel{} follow a common pattern, using a URI to identify a \codeword{type} followed a \codeword{spec} section to fill in the details. @@ -2379,7 +2511,7 @@ \subsection{Type URIs and specifications} to resolve the URL into a description. \subsection{\execbrokerclass{} request} -\label{datamodel-request} +\label{sub-datamodel-request} The main section of the \execbrokerclass{} \datamodel{} describes the \codeword{request} block, which describes an \executablething{} and the resources needed to execute it. @@ -2393,7 +2525,7 @@ \subsection{\execbrokerclass{} request} \end{itemize} \subsubsection{Executable} -\label{datamodel-executable} +\label{subsub-datamodel-executable} The \codeword{executable} section of the \codeword{request} block describes the \executablething{}. The core \datamodel{} simply defines this section as containing a URI identifying the \codeword{type} of executable, @@ -2416,7 +2548,7 @@ \subsubsection{Executable} by defining their own types of \executable{}. \subsubsection{Jupyter notebook} -\label{datamodel-jupyter-notebook} +\label{subsub-datamodel-jupyter-notebook} The \datamodel{} extension for a \jupyternotebook{} defines the URL to identify the \codeword{type}: @@ -2456,7 +2588,7 @@ \subsubsection{Jupyter notebook} \end{lstlisting} \subsubsection{Zeppelin notebook} -\label{datamodel-zeppelin-notebook} +\label{subsub-datamodel-zeppelin-notebook} The \datamodel{} extension for a \zeppelin{} notebook defines the URL to identify the \codeword{type}: @@ -2496,7 +2628,7 @@ \subsubsection{Zeppelin notebook} \end{lstlisting} \subsubsection{Docker container} -\label{datamodel-docker-container} +\label{subsub-datamodel-docker-container} The \datamodel{} extension for an \dockercontainer{} defines the URL to identify the \codeword{type}: @@ -2544,7 +2676,7 @@ \subsubsection{Docker container} \end{lstlisting} \subsubsection{Singularity container} -\label{datamodel-singularity-container} +\label{subsub-datamodel-singularity-container} The \datamodel{} extension for a \singularity{} container defines the URL to identify the \codeword{type}: @@ -2569,7 +2701,7 @@ \subsubsection{Singularity container} \end{lstlisting} \subsubsection{Resources} -\label{datamodel-resources} +\label{subsub-datamodel-resources} The \codeword{resources} section of the \codeword{request} block describes the resources needed to execute the task. @@ -2598,7 +2730,7 @@ \subsubsection{Resources} \end{lstlisting} \subsubsection{Compute resources} -\label{datamodel-compute-resources} +\label{subsub-datamodel-compute-resources} The \datamodel{} for a \codeword{compute} resource contains: \begin{itemize} @@ -2629,7 +2761,7 @@ \subsubsection{Compute resources} resources. \subsubsection{Generic compute resources} -\label{datamodel-generic-compute} +\label{subsub-datamodel-generic-compute} The core \datamodel{} defines a generic computing resource that has cpu cores, memory and a hierarchical filesystem. @@ -2710,7 +2842,7 @@ \subsubsection{Generic compute resources} \end{lstlisting} \subsubsection{Storage resources} -\label{datamodel-storage-resources} +\label{subsub-datamodel-storage-resources} \begin{lstlisting}[] .... @@ -2718,7 +2850,7 @@ \subsubsection{Storage resources} \end{lstlisting} \subsubsection{Authentication} -\label{datamodel-authentication} +\label{subsub-datamodel-authentication} \begin{lstlisting}[] .... @@ -2726,7 +2858,7 @@ \subsubsection{Authentication} \end{lstlisting} \subsubsection{DateTime} -\label{datamodel-datetime} +\label{subsub-datamodel-datetime} \begin{lstlisting}[] .... @@ -2734,15 +2866,15 @@ \subsubsection{DateTime} \end{lstlisting} \subsection{Response} -\label{datamodel-response} +\label{sub-datamodel-response} \subsubsection{Positive response} -\label{datamodel-positive-response} +\label{subsub-datamodel-positive-response} A positive (yes, this platform can execute the task) \execbrokerservice{} response contains \codeword{result} set to \codeword{YES}, and a list of \execoffer{s}. \subsubsection{Offer} -\label{datamodel-offer} +\label{subsub-datamodel-offer} Each \execoffer{} contains a unique identifier a status value and an expiry time for the \execoffer{} followed by an updated copy of the \codeword{request} with details @@ -2773,7 +2905,7 @@ \subsubsection{Offer} \end{lstlisting} \subsubsection{Negative response} -\label{datamodel-negative-response} +\label{subsub-datamodel-negative-response} A negative (no, this platform cannot execute the task) \execbrokerservice{} response contains \codeword{result} set to \codeword{NO}, followed by an optional list of reasons @@ -2797,7 +2929,7 @@ \subsubsection{Negative response} %TODO more examples of reasons ... \subsection{Session} -\label{datamodel-session} +\label{sub-datamodel-session} An \execworkerclass{} \workerjob{} record consists of a unique identifier for the \workerjob{}, and a \codeword{phase} indicating what execution stage the \workerjob{} has reached. @@ -2821,7 +2953,7 @@ \subsection{Session} \end{lstlisting} \subsection{Extensions} -\label{datamodel-extensions} +\label{sub-datamodel-extensions} \begin{lstlisting}[] .... @@ -2829,7 +2961,7 @@ \subsection{Extensions} \end{lstlisting} \subsubsection{nvidia gpu} -\label{datamodel-nvidia-gpu} +\label{subsub-datamodel-nvidia-gpu} \begin{lstlisting}[] .... @@ -2837,7 +2969,7 @@ \subsubsection{nvidia gpu} \end{lstlisting} \subsubsection{Xilinx FPGA} -\label{datamodel-xilinx-fpga} +\label{subsub-datamodel-xilinx-fpga} \begin{lstlisting}[] type: "https://www.purl.org/ivoa.net/resource-types/xilinx-vu19p" @@ -2850,6 +2982,7 @@ \subsubsection{Xilinx FPGA} \pagebreak \appendix \section{Changes from Previous Versions} +\label{sect-changes} No previous versions yet. % these would be subsections "Changes from v. WD-..." @@ -2858,7 +2991,7 @@ \section{Changes from Previous Versions} \pagebreak \appendix \section{Resource type URLs} - +\label{sect-type-urls} https://www.purl.org/ivoa.net/executable-types/example https://github.com/ivoa-std/ExecutionBroker/blob/main/types/executable-types/example-executable.md diff --git a/notes/zrq/20241128-01-gardening.txt b/notes/zrq/20241128-01-gardening.txt new file mode 100644 index 0000000..5b11d4e --- /dev/null +++ b/notes/zrq/20241128-01-gardening.txt @@ -0,0 +1,200 @@ +# +# +# +# Copyright (c) 2024, Manchester (http://www.manchester.ac.uk/) +# +# This information is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This information is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +# +# +#zrq-notes-indent +# +# AIMetrics: [] +# + + Target: + + Success + + Result: + + Work in progress ... + +# ----------------------------------------------------- +# Create a new branch. +#[user@desktop] + + branchname=gardening + + source "${HOME:?}/execbroker.env" + pushd "${EXECBROKER_CODE}" + + newbranch=$(date '+%Y%m%d')-zrq-${branchname:?} + + git checkout main + + git checkout -b "${newbranch:?}" + + popd + + +# ----------------------------------------------------- +# Build our document. +#[user@desktop] + + source "${HOME:?}/execbroker.env" + podman run \ + --rm \ + --tty \ + --interactive \ + --name ivoatex \ + --volume "${EXECBROKER_CODE}:/document:rw,z" \ + ghcr.io/ivoa/ivoatex-docker:latest + + make clean + make + +# ----------------------------------------------------- +# + + Move 'Session lifecycle' to a sub section at the start of 'Execution broker' + + Add hint at the start of each code example. + + [+# OpenAPI specification+] + [+# Executable metadata+] + [+# ExecutionBroker OfferSet request+] + [+# ExecutionBroker OfferSet response+] + [+# ExecutionBroker Session response+] + + +# ----------------------------------------------------- +# + + 1 Introduction + 1.1 Role within the VO Architecture + 1.2 Supplementary documents + 1.2.1 OpenAPI specification + 1.2.2 IVOA profiles + 1.2.3 IVOA Single-Sign-On + 1.3 Executable things + + 2 Service interaction + 2.1 Discovery services + 2.1.1 Software discovery + 2.1.2 Data discovery + 2.2 Execution Broker + 2.2.1 OfferSet request + 2.2.2 Resource allocation <-- move later + 2.2.3 Execution scheduling <-- move later + TBD + 2.2.4 Offerset response + 2.2.5 Session lifecycle <-- move later + 2.2.6 Session execution + 2.2.7 Update options <-- move later + + 3 The data model + 3.1 Metadata roles + 3.1.1 The developer + 3.1.2 The packager + 3.1.3 The publisher + 3.1.4 The user + + 3.2 The executable + 3.2.1 Jupyter notebook + 3.2.2 Docker container + + 3.3 Resources + 3.3.1 Compute resources + -- truncate the text about URLs + 3.3.2 Storage resources + + 3.4 Date and time + + 4 Federated architecture + -- drop all this for now ? + -- new stuff about proxy and aggregator services + + 5 Example use cases + -- all of these need details + -- need to restructure + -- fix the levels + 5.1 Simple notebook + 5.2 Notebook with dates + 5.3 Notebook with data + 5.4 Notebook with compute + + 5.5 Simple container + 5.6 Container with data + 5.7 Container with compute + + 5.8 Kubernetes Helm chart + 5.9 Spark cluster + + 6 Service specification + 6.1 General requirements + -- replace with supplimentary docs ? + + 6.2 ExecutionBroker + 6.2.1 Request method + -- change to offerset GET and POST + 6.2.2 Offer GET + 6.2.3 Offer POST + -- move the options here ? + + -- everything below eher gets moved up or dropped -- + + 7 The data model + 7.1 Type URIs and specifications + -- simplify + + 7.2 ExecutionBroker request + -- fix the hierarchy + 7.2.1 Executable + 7.2.2 Jupyter notebook + 7.2.3 Zeppelin notebook + 7.2.4 Docker container + 7.2.5 Singularity container + 7.2.6 Resources + 7.2.7 Compute resources + 7.2.8 Generic compute resources + + 7.2.9 Storage resources + -- blank + + 7.2.10 Authentication + -- blank + + 7.2.11 DateTime + -- blank + + 7.3 Response + 7.3.1 Positive Response + 7.3.2 Offer + 7.3.3 Negative response + + 7.4 Session + -- junk + + 7.5 Extensions + -- blank + 7.5.1 nvidia gpu + -- blank + 7.5.1 Xilinx FPGA + -- blank + + + + +