From 9fb8c433d21b00ec5f3f48e27183492b0ab1eabc Mon Sep 17 00:00:00 2001 From: Maximiliano Osorio Date: Mon, 1 Jun 2020 19:42:33 -0700 Subject: [PATCH] Add: encapsulate process (#111) * feat(mic) component init * fix: create a dummy config * add jinja2 as dependecies * fix: init * fix: develop docs * add develop docs * fix: typos * typo * renamed some steps. Changed step 0 to glossary * improve docs * fix: reorder * skeleton git push * adding new structure * fix: ideas * fix: typo * step3: create skeleton * test skeleton * add: test skeleton * Add: Step4 create yaml (#78) * add skel command * adding test * create yaml * fix: step_4 without test * adding tests * adding tests * add test with directories * fix: improve the usability * fix: add pyyaml as dependencies * fix: cleaning filenames * Step4: Add the YAML variable in the run (#79) * fix: step 6 - generate run.sh * add: step3 * add: test step4 * fix: add help text * Add: Step5 local execution * add: local execution * fix: ignore windows * fix: add zip format * Execution docker + outputs (#85) * add docker build * add: replace files * fix: cleaning the code * fix: replace in execution * step7: running using docker * add: status steps * fix: improve docs on cli * Add integration with GitHub (#80) * modelconfiguration publish will make repo if it does not already exist * Added begining of GitHub publish functionality C:\Users\Admin\Desktop\USC\Internships\ISI\mic\mic.zip will now * Fixed quotation marks typo * publish will check parameters before uploading zip file Checks if file exists before uploading. If file exists it will check if anything has changed before updating * Publish now uploads model's READE.md to GitHub Looks for README in given directory. If it cant fine one it will make a new README.md file in the repository * added error if github credentials (Token) are incorrect * Can now configure GitHub credentials in mic configure * add --force (-f) flag * Add a check to make sure README wont be overwridden accidentally Also made -d option actually work * Added documentation on GitHub credentials * Added functionality for github releases * Lower case J, maybe this was failing the test * Added tag and message options for publish command * Better error handeling + tag and message options * Better documentation for generating GitHub token * Added some better details * Updated method documentation Co-authored-by: Maximiliano Osorio * Publising (#87) * Revert "Add integration with GitHub (#80)" This reverts commit bf9c54e5e4375a11282dda9350edbcfa63f452f1. * add: publish docker and git * Publish (#96) * Revert "Add integration with GitHub (#80)" This reverts commit bf9c54e5e4375a11282dda9350edbcfa63f452f1. * add: publish docker and git * fix: better message * add: handle outputs * Improve documentation and software dependencies * Fix: Added pygit2 to setup.py * Fix: credentials swaps users email and name * Docs: Updated documentation for configure * Docs: Fix typo * Fix: Add PyGithub to setup.py * remove email duplicate Co-authored-by: Maximiliano Osorio * Mic ch (#99) * Fix: Added pygit2 to setup.py * Fix: credentials swaps users email and name * Docs: Updated documentation for configure * Docs: Fix typo * Fix: Add PyGithub to setup.py * Add: step1 creates .gitignore * Model catalog push (#100) * push to model catalog * fix: push to model catalog * fix: extract zip * fix: use basedir pwd * fixing: step1 * fix: bug step3 * fix: bug step4 render output * fix: verify if outputs is none * fix: default value is 0 * fix: removing the local execution * fix: removing the option add a new mc to a model * Add version mdodel (#104) * fix: add imports * fix: select existing model versions * Mic ch (#102) * Fix: Added pygit2 to setup.py * Fix: credentials swaps users email and name * Docs: Updated documentation for configure * Docs: Fix typo * Fix: Add PyGithub to setup.py * Add: step1 creates .gitignore * add: list_credentials mic list_credentials will show the user their different profile configurations. This will aslo help with debugging issues with credentials. This command will not show password or full github token for configuration * Add: --short flag for list-credentials Short parameter allows user to clearly see a list of the profiles they have configured * fix: typos and minor changes * Fix: import typo * Fix: step4 and pytest typo * Fix: test_executor no longer crashes * Add: append comment to default parameter * Fix: write_step() deletes yaml comments * Fix: Replace yaml.dump with write_to_yaml() write_to_yaml() makes sure that the comments on the yaml file persist when using yaml.dump since yaml.dump will not save any comments made * Add: description field to input, parameter and output fields in mic.yaml * change: config.yaml -> mic.yaml and mic configure -> credentials Co-authored-by: Maximiliano Osorio * Add version model (#110) * fix: add imports * fix: select existing model versions Co-authored-by: Daniel Garijo Co-authored-by: Christopher Heidelberg --- .gitignore | 2 + .travis.yml | 62 +++- docs/configure.md | 78 ++++- docs/figures/tutorial/01.png | Bin 0 -> 27717 bytes docs/glossary.md | 6 + docs/model_configuration/02-pre-steps.md | 37 ++ .../model_configuration/03-create-skeleton.md | 12 + docs/model_configuration/04-copy-your-data.md | 77 ++++ .../05-write-invocation.md | 61 ++++ .../model_configuration/06-pass-parameters.md | 92 +++++ docs/model_configuration/07-validate.md | 26 ++ docs/model_configuration/08-outputs.md | 43 +++ docs/model_configuration/09-publish.md | 10 + docs/model_configuration/docker.md | 1 + docs/usage-add-configuration.md | 3 + mkdocs.yml | 10 + pytest.ini | 6 + setup.py | 7 +- src/mic/__main__.py | 331 +++++++++++++++++- src/mic/_makeyaml.py | 90 +++++ src/mic/_schema.py | 66 ++++ src/mic/_utils.py | 26 ++ src/mic/cli_docs.py | 5 + src/mic/component/__init__.py | 0 src/mic/component/dame.py | 0 src/mic/component/executor.py | 202 +++++++++++ src/mic/component/initialization.py | 119 +++++++ src/mic/component/python3.py | 10 + src/mic/config_yaml.py | 313 +++++++++++++++++ src/mic/constants.py | 62 ++++ src/mic/credentials.py | 62 +++- src/mic/file.py | 13 +- src/mic/model_catalog_utils.py | 6 +- src/mic/publisher/__init__.py | 0 src/mic/publisher/docker.py | 29 ++ src/mic/publisher/github.py | 221 ++++++++++++ src/mic/publisher/model_catalog.py | 214 +++++++++++ src/mic/resources/model.py | 17 +- src/mic/resources/model_configuration.py | 3 +- src/mic/resources/software_version.py | 25 +- src/mic/templates/.gitignore | 2 + src/mic/templates/Dockerfile | 16 + src/mic/templates/io.sh | 109 ++++++ src/mic/templates/output.sh | 3 + src/mic/templates/run | 29 ++ src/mic/tests/test___main__.py | 181 ++++++++++ src/mic/tests/test__schema.py | 9 + src/mic/tests/test_directory.py | 4 + src/mic/tests/test_executor.py | 29 ++ src/mic/tests/test_initialization.py | 6 + src/mic/tests/test_model_catalog_utils.py | 6 + tox.ini | 5 +- 52 files changed, 2702 insertions(+), 44 deletions(-) create mode 100644 docs/figures/tutorial/01.png create mode 100644 docs/glossary.md create mode 100644 docs/model_configuration/02-pre-steps.md create mode 100644 docs/model_configuration/03-create-skeleton.md create mode 100644 docs/model_configuration/04-copy-your-data.md create mode 100644 docs/model_configuration/05-write-invocation.md create mode 100644 docs/model_configuration/06-pass-parameters.md create mode 100644 docs/model_configuration/07-validate.md create mode 100644 docs/model_configuration/08-outputs.md create mode 100644 docs/model_configuration/09-publish.md create mode 100644 docs/model_configuration/docker.md create mode 100644 docs/usage-add-configuration.md create mode 100644 pytest.ini create mode 100644 src/mic/_makeyaml.py create mode 100644 src/mic/_schema.py create mode 100644 src/mic/cli_docs.py create mode 100644 src/mic/component/__init__.py create mode 100644 src/mic/component/dame.py create mode 100644 src/mic/component/executor.py create mode 100644 src/mic/component/initialization.py create mode 100644 src/mic/component/python3.py create mode 100644 src/mic/config_yaml.py create mode 100644 src/mic/constants.py create mode 100644 src/mic/publisher/__init__.py create mode 100644 src/mic/publisher/docker.py create mode 100644 src/mic/publisher/github.py create mode 100644 src/mic/publisher/model_catalog.py create mode 100644 src/mic/templates/.gitignore create mode 100644 src/mic/templates/Dockerfile create mode 100644 src/mic/templates/io.sh create mode 100644 src/mic/templates/output.sh create mode 100644 src/mic/templates/run create mode 100644 src/mic/tests/test___main__.py create mode 100644 src/mic/tests/test__schema.py create mode 100644 src/mic/tests/test_directory.py create mode 100644 src/mic/tests/test_executor.py create mode 100644 src/mic/tests/test_initialization.py create mode 100644 src/mic/tests/test_model_catalog_utils.py diff --git a/.gitignore b/.gitignore index c031522..9c028b3 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,5 @@ +mydir/ +test Scripts/ *.json pyvenv.cfg diff --git a/.travis.yml b/.travis.yml index 717f49d..5748e06 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,9 +1,61 @@ -dist: xenial language: python -python: -- '3.7' -install: pip install tox-travis -script: tox +matrix: + include: + - os: linux + python: '3.6' + - os: linux + python: '3.7' + - os: linux + python: '3.8' + - os: osx + language: generic + python: '3.6' + before_install: + - brew upgrade pyenv + - brew install pyenv-virtualenv + - export CFLAGS="-I$(brew --prefix openssl)/include" + - export LDFLAGS="-L$(brew --prefix openssl)/lib" + - pyenv install 3.6.7 + - eval "$(pyenv init -)" + - pyenv virtualenv 3.6.7 venv + - pyenv activate venv + - os: osx + language: generic + python: '3.7' + before_install: + - brew upgrade pyenv + - brew install pyenv-virtualenv + - export CFLAGS="-I$(brew --prefix openssl)/include" + - export LDFLAGS="-L$(brew --prefix openssl)/lib" + - pyenv install 3.7.2 + - eval "$(pyenv init -)" + - pyenv virtualenv 3.7.2 venv + - pyenv activate venv + - os: windows + language: sh + python: '3.6' + before_install: + - choco install python --version 3.6.7 + - export PATH="/c/Python36:/c/Python36/Scripts:$PATH" + - os: windows + language: sh + python: '3.7' + before_install: + - choco install python --version 3.7.2 + - export PATH="/c/Python37:/c/Python37/Scripts:$PATH" + allow_failures: + - os: windows +install: +- pip install coverage +- pip install codecov +- pip install -U pytest +- pip install -U pytest-cov +- pip install --upgrade setuptools wheel +script: + - python setup.py install + - pytest +after_success: +- codecov deploy: provider: pypi user: __token__ diff --git a/docs/configure.md b/docs/configure.md index 3edf65c..844d31f 100644 --- a/docs/configure.md +++ b/docs/configure.md @@ -1,13 +1,85 @@ -## Login and credentials -The MINT Model Catalog requires credentials for modifying the contents in the catalog. Use this command to configure username and password for the [Model Catalog API](https://model-catalog-python-api-client.readthedocs.io/en/latest/endpoints/). +## Overview +``` +mic configure [-p | --profile] [--server] [--username] [--password] [--name] + [--email] [--git_username] [--git_token] [--dockerhub_username] +``` + +## Description + +mic uses several APIs to upload models. The MINT Model Catalog requires credentials for modifying the contents in the catalog. Use this command to configure username and password for the [Model Catalog API](https://model-catalog-python-api-client.readthedocs.io/en/latest/endpoints/). This command can also be used with no parameters, it will prompt the user to enter any required field not given. + +## Options + +`-p, --profile ` + +Credentials can be set up with multiple configuration profiles. This option lets the user choose which profile they are editing. If the profile does not already exist it will generate a new one + +`--server ` + +The Model Catalog API - [required] + +`--username ` + +Email for the Model Catalog API - [required] + +`--password ` + +Password for Model Catalog - [required] + +`--name ` + +Full name of the author - [required] + +`--git_username ` + +Author's Github username - [required] + +`--git_token ` + +Authors's GitHub API Token. More information can be found in the [setting up GitHub credentials](#GitHubCreds) section below - [required] + +`--dockerhub_username ` + +Username for dockerhub + + +## Setting up GitHub credentials + +GitHub credentials are also required for mic's GitHub features + +The `GitHub Username` field is the users GitHub username. If unknown the username can be found at [GitHub.com](https://github.com/). Once logged, in at the top right dropdown menu there will be a "signed in as **[username]**" + +The `GitHub Token` is the user's [personal access token](https://help.github.com/en/github/authenticating-to-github/creating-a-personal-access-token-for-the-command-line). To create a personal access token click [here](https://github.com/settings/tokens/new), or go to GitHub.com -> Settings -> Developer settings -> personal access token. Click **Generate new token** this will open the new personal access token page. The following options must be checked: + + - [x] repo: | Full control of private repositories + - [x] write:packages | Upload packages to github package registry + - [x] read:packages | Download packages from github package registry + + Writing "mic access token" under notes is also recommended + + When done click **Generate token** at the bottom of the page. Once the token is generated be sure to copy and save it in a secure location. Enter this key in the `GitHub API token` field when prompted. + +!!! warning + If this token is lost there is no way to recover it without generating a new one. + +### Setting up DockerHub credentials + +### Example usage: ``` $ mic configure Model Catalog API [https://api.models.mint.isi.edu/v1.4.0]: Username [mint@isi.edu]: Password: +Name: +Email: +GitHub Username: +GitHub API token: +Docker Username: ``` + + !!! info - [Contact the MINT team](mailto:mint@mailman.isi.edu) to create a new user/password if you want to edit your own models. + [Contact the MINT team](mailto:mint@mailman.isi.edu) to create a new user/password if you want to edit your own models. \ No newline at end of file diff --git a/docs/figures/tutorial/01.png b/docs/figures/tutorial/01.png new file mode 100644 index 0000000000000000000000000000000000000000..bfcce60136ff71cc32a265b187469c0ff18f6023 GIT binary patch literal 27717 zcmd42Wl$W^);5YIxVw7@ZoxGW+!9D|cXxMp3BjES?(QBE+}+*XUA`uA&RgHDTlN0A zRd=eVW_EY)-fP>EXFVPAQC18Q4i63t3=C01{DV9g82BCVRfl;EoFVz>-UtSUAZ{Wo z{82(!nB=3am7$5b0T`HgNSqS%XN6w8bTxSiV;>lC*p?77@dPxXFcOk4<*45Tg+iD~ zgJ+^eUQIKQX?KRcZNw6s3T>eaU991y%YVDC+Y#88f-y3AzQl7ZV07euo#EBmroUzQ z8Z6B;{d2P49>iDL-k4tj&{bWz*_Gl^;IFL(q25|=pKi)Knww*Ik1andEG>csSGMha z-?(#mXbVjje7g><@{co)QR_)ORR9KS#y(uT}4V@#&aJ_wn^*(6d> zaZ6^&4Wk6vkOtC1(W7kJpue-jhS>z2;tiS3Zqz4wrjo9gv)S#tu*7NQJW7Z-lBsoi zNl|QEoww0rP*N1>b>-i#Wg7?AkQ(+?t&>l(C(3|=(HE`A6@F5$*ZIsSy@f?qJl8{K z)z`7Xb5kj87SJ;H5g&+{TIu8cB;+4xM3s>jx(u`S2IQYMGfW;E6|pxgLV^JMWjj&> zOA?LkR`+nMd4+RuVR|f8If73XbIYIhQV}Jk?|hxj?$n#No@DaDx%wR=`+GvF^z>i& zm_?D$j9{O$VVgb_Mol0}0wjyHkK^yneeHnll31T1NA^PBB+G zb+_kK1?Y`r)~Ac}O8Yzd5YSH)T`1=8o#2!e@OomRR~hfi8&;R<<-Dqw4`AhBEc}D~ z_x&Az97?Q_%_85Hw;PWj@%fwB%0BZOEg)68u5{E3Yy>TEqH};V+`#OFL_iBI6MgI- z{ZKmaBzEMraiTHF=7^yLCyDz3_B-bTr^39%yq>p+%J?yDb|jleW@)NVH1VTdYf&}W zRkVPrE_}Ji4DJD^BNC548w8dkhS?7W-Jm#UZA?DJlJA~YHigcWq)|6iUc3=<29%qn z4y9?O8IyW0KOi=*QLW!d<@Nqd`p%1m?|BuIX7L4HWr&7<^B)9r42^h_Y4^JOXBBa?0 z*trd4dOI?d&qpS>pWaYkA>RC;30rZE_bJeRod_=}h^q{z=p$K9WDYUk!BCF<&R2Ma zlM|7_yMKkd9#TmT+8MmDgJ&d+1O^sFj_?zwLeL!5=qKW-D8$!~6mTKJ@?YOkphuIy z>Az+{rWTGTkLwZ5#=1r13MLR6BDLtT*Z?03X%^)N;YbQ2eq^0MA`EoRGJl6ABS@8* zw1ZKNLNAJ)g=vh~h;AVmm4!XBZ_INCao7Qt8^FvgHSp@Zi&GgPKGZ;)#T-j9G)|9A znQscgYr9C9XDYzsX9y>x(JF)WE3uUjPT~yT;UCzmaW{_5-~u69zlc{BZFYLl)gbF^Ar=Vku2(h?<9vjXI6ygdT-D9~>3T zM}gLxxFLN;_3Nv1Hm^La38D$r?%Q3s0>2BNVv$;srs&R|0)5zqV0GE0_xwVwLY|UG zIXm*^)UB~Nq24SJmHw4UW6}*Db~7*JFY~WHdznJ8g>fgSzAwp3|Gt#dIyyF1Id(RR zmqj_1ob?4%Beq{W1z8gH&LbH;LKt68VFYWub~JL-a(roI*Ph1}g)c5$OiQ{fCnIw? z&uhfc7SKb4PVS%*+(X=GVExy2Z@SD$C4` z5d8Y;wYndjAB&&H>yp=tfp>xA2r&ro2t^3U__eIexr8zO13C;kM+QsUo_PA~!F=MJ z#;cik@0DJ+aNwC^=u^C-VVkqCU)+gd?&qA~oOe#L$vWjjU-;RpAKv>p#^Ft#bmMD$ zge&|3Vs&ok;}rp!PYP7yoa4K>DbkF|SBXg}0VxuLQ;BIz`z%Ju&`Cx~W$y%ZmrOd! zVaid|imN$IPg4zgbgOKYF^gl0Eu|FtVWa{!Mf(-|HEG?7J`|A^Eh=6Y7fdZp;ZD(( zs1-L^(3y+Q50n>F-HylW?px4SYFM~+UA3gOXjCq`H;FbGU5Q?)-wWMy-!~!I2MGpI zBe5Y(6NE%AMb1W=5i;_CWQk* zU_cIJ2@*x5$D8B4*knr4Dd>zxAVbz!qa@$>~dqqLB-Lk7B|o4 z)aaIYZ+`veT8TVW3`VRN-=^Phm3dCDRV=IDK+?@?!9OasR$9BTTDy8L7>P{7?)|%#!G~XSpw9yKS_M&l~eARyk=V z-w4ZBY-RKtrXFr_lxnse>iOujUrW87xCeN2@cbwb>FfPTy3mU8aVzN7-b{!0Z%^4aiMd21bM5I<(6jJYvYhF}D#X$i zy1e6do!G(5Iu@N5mHLjaCy2mdC^FEZd_O~7(H2D-8A7PQwk5G}Qjcq6w1#^cv*z@( zO$tU<`I|M>DCwu@nQz%K@AwPYcRg&y&(1ckDmuG{2;4}2QHZ3D#Y#%bNG>MNu`<{% z6xtB3^K~VJw=s8k3U&C~q07A4lIG==Fha8$YkeoAYC9KMJ=~0=iC9LJmZF)qG`M}D zbgl*C3q|`XO~`B2c6A)#IeZg$nB|dGN#8LPLS*wh1GAVqwPze=bPfN$hokgW?Llkm zM(Se2#zOUiADV|m?Z$@wdGD*3`Rt)FG?RObW$L0_`+hP5was96-)8DzFrpbo8Ul>A zw?rGz8%mpAiOKIAw0tyuvRBsL<(a_e@zEpDF0c@03}q~_blP?;;36AV8d;_W3`k~7 zvdp_Wo*qPUt=iQN=L`*|--|_?jh=n4JO8%LhMj^rOzn9gb;IaDIY(LfE#>@it4zzA z5rxrMW2e=6@JCsW@7TFPsllc}2KFg-+Oxw=8D`rR{gCoP<4?QZ#+;uy-*PzOUop(L zs$03No{7irvO6jna zy+l*_(X&MRFwWuq{@!#-38UJxiJx<4NyI8XzUhH!9|tPOrj=_Ym&th&{gRq#U9vg- z{GCCHzS{j&{UMcEi>aAY;*qj^&=mC^w^QV@i;GfEX|8L+T|`q-Q~#AWUnt+Om(=}X z*HTXl{eAQG^m5!p^{; z1OgBdHUF4X^@;H?^KM1ydQvxWjkIZlo$f)18C7?xIUlCW-obA9IMq=3Fah7b*VPEu zvK{oWTZUEhiVJAn)o;@|x&8bJDf6PWgbirUn>UkRi}K2B$~5YU zJX#J#*CNz7x|hw{DC;i|9ET2-ytbZa-He7bek_i;HD31}r!1A-CakwP*16RZd!n4J ztaaW5IT4)_w>Z;zJv3z9z&=XcDum_L3@@Y;?jh_qKe#WPM`Dj*w`fp$?LK~a99!Ld zYV&()o#Cr(T8}&Uvb-;16~1il#yPZ=VKo3o z!UlOFGBrKjR9kl3Z-{eX0TwCqX_+xj$lu8iv5yC}zQSRZ;Kc6%|7_CZ6{m6s6cAd# zT8pdLfq`LBzI?$Y!+|rKQnUC!E9o)eGm(z@7B)@O5H{&Bymib5`Y-MXe!p889fr*SCj)a7S*H+(< zTmFOS-|9e#kIdNK-kO_{(aFh)!HJc@%GQXHnTv~yk%@(og@qotgWk@?(q6}z-qMcz z&qMw_=YxTro~?bXs9xHV8~U$bANWAzr2i_>={?(AEZGiVu;*zwNWyvd__Qmx)k&kA8FHW_f8LW4n!BRjZb}V zt8A<3`!=8AnDf)=$6@pV2v=Wlk?@UFyFjnD8pZkXc|IPHQJ#~>zRS#Rsd^jHR>{d1 z2cgBrOGbD?*sox}55ZiDuY#cA*x|ScGH^7KmqWx`XqXY=?+yC85EysT57x8q{#-;8 zH=Zdco!+_S`+YAOiI23Tw@`)zFh2+!`@<6ESrFPDwt%VgXNI_v=NcR{y;Hlv)3$IHmadGtJrqHkZpuC5_H7Y(@qu+0>8E>g2ba zE2Cn>(PBg{6yNdI+?won$I8d$cREG8pn&1rm1(y)eB4aXlJ9bsN#Q?K&H<{vzEKOP zG8xa?c)CAMs4^PvtG8L-3?lY~2~J7p7_A+q_b6wdnS~15cfd*aTU41xDb^b=P!KCn z$cuD;IPI=yg54k^SC}hk}&!eZ-#8fOx!B+vC)?6fefZr&yDq7CBbn>-f* z-JdYjpT5_-96MGR4HMz=ItRmbA-69y3@3dOxZ5FDsWz?vuIbR5Z4YU%n9nfPKmw}aFKR5`PI38QeU842r&TL#@A`DoBxVMyrJ8suUaqs1o~g-DI;1Yk^W(dT7L<{NY?qb!U3iWvlDu z-dKa3HogOAuFdtqEG;m-@z=}Gf{wr)emA?q5UWooT)(=VP}*28_ouP#3&kSv=u|@up`MW)Tc72BDT=>1m=%r~5_mWvZ#b+~n{-eWiefogz~iu_i{iV= zr$MX5ihke^R9H4%7ro~tmpL{Awe6QSWjpLEzM1E;+v*V`_Ap=ChYy|sw%J9A`{^%i zN#*M9^WovRW|#9lwdvs)DmgseNyUIHrv~R!GFFD#Rp~^gj%KH(@_2`bo71oU&@MW6 zS?6B;q-*<6`*T(@X_M&J_zx1_k4gX7Q?(>?&Loox?WJs3XA<;tYIfyZhlxB?fC z;yvTgTfG0&aqRT`1i8`f0~t*cB7t|$D%qd=K}kr!%cD%d&x6>WAa6KRXu1AW_#3?% zdnbf#lEU2AM<%j^v79Q>R4M)asS?t>s%H`-MNr#rZsS;q+KvF-{&X1!HzB(4)(lA4 zEi2DG1=s6f#s)lVeLFoR>#1rNCb1=vbvtS{oQSWHb{(5=sm@AZw$-Dl@o=stY{C6} zko)bR+&wFT*iQ6qUWVrzR--3oH*_h`bi{0>K^OuK)5qnraj`#xF>dM`Q=S{w6=tm= zpwaqBWDxuIlhEDeeo<~&7CmdaP?XBgd8;;=L+5s5f~pYNLz$uo9P^9Ac`?(8d=U56 z`PQHutwf`L_m4(?G=AgaqLKCfcRD|uC!17j%~gKtMlH={z8{uMVljyQvLB2}E*wYz z59;P#m?%&n-1jzFTlTz{37CELR*~G*U1EFCts_-cY_*&$jn^f#5I*-@t=gEO#%vlX z->%K`(O|yT0^HM*<4fUZ_Vf-CR-CNQj~S#!L#bzDQaXU?oFy3t`@vcvb)Ad3yN z#~p%7m7$C`vDKO!$pSGyp+HHp8AygUeR{Rgv(f$D!w2FORKK4{7}ja_jx#J@_>tjo zdd@|TSQJsEl~!M6dP@t49qmebqzE3H0f&`UV`HX55A-HNqS^R^rczIDBfmUp<>J!ydtWLGy8{ypoY+EBqYW!6N3bpHGkyY@|Bza)2vy&i08GL!_=gLyw-`5HPf zDKm>hkIpJXXFj$hr&J_lvWR7zi=98O8OU4w7c7$%(Py zv+ids!N(y{bfbMu_Lk$0gHYx0iWx;1+vc%&Sj&^ z9@mRwgb!`BN)$eV#NI@Q@`--;gGNRp zutW*z$vP158;K&|5~5Ko4&I_M%21qsVzPES{dHS7{z~9NJ;Spx2}$7IC#H125@BHdM_7pI4Xk)QooAeN;rCDS8A33Tg9Wis&TzEf3>`7C5OdoH z??xoS42ZV}kX`JIOccBLB`t9`M#ha~X_nuQ915?}tJT_?XQ(i1X7hj-@2;tU^!5V6 zE`lb)Dx-s1+}-Z2qi{Ls7CU;_<~z?vAy{CX2E?6pFg^Q0|&1Jb)?A zD%0+AMrpvLU8*z!9YrJUS6G8!zJ?m#qn9!-rE*oh8n>}l1g|wHvOG+KkOq;|v?&)%-gwxe_ns_?v0 zCNr65H^Ayhj~X8cVWT(1Pv)@FspU}=igIjcd5%}JWKN0IXO({k=^SkW^Wjc-57YaC zW?4qO*;GvRRy&!nDU8V?N`pa@lMXqGSU{E@(qHdz)Gp~v}OI;WtixQbwqBq+eS5n#dzQ;_1e zm1Kmle5SUIw{a1eN&pH9Im+38p0HsL+1 zCuCn0yY2u4`D`K?q#dh}=7WV&P5{k9caE z%XZ^q8`X`*>5tt-r5UpM?;=u+mEoMf*L2jyJ!<7h)-RI~zT92Xv&7mc#f?u(8Si%z z=v2({stCF5#nWAKS(A@;tZgR%7s@hF=MD+$2E;9Wzsq+d4&B+HSfX06A@CetVkSIL zAnd#*01u^VC9_bj<41Z#J$}RV6`y4z>hVT)|9r|-CWU=cJ+C`|L1w4Kr`5GN@3SF6ezpM8ESDYL&GHm9M8X}b760@<{6%m zw=7&s&9$E3b*I$<37moVri0tTtw{egy+`(PRJlJpnW@o^K$Sxw}HAfE4aU2;WK zp|YPh`{D+5nI9uePCgPAJJ5cStbpb;CuoQCYc2dzrqs(E+^wfUgcN^ULcfhO zkjA6Lf3qTtX;?QE1o*_};v(gbNdjK2E-)yL{5wd2X;t(|E@8MqtseKe$y;GLBXwre zifM_tw5x^!pw8Q*Cn&PVRlA~s>Dz|5^{oue%_i6owU*|qX?9`dqU9P|TQx|tmU4LG zvki&9z>Y?<9z3d+(nI&d``yJo(Dm^A`2J0_r@#*^SBzk6IPK@^+OFq^bH(#-gsXpX7Q#B@%RWgo%?e8jzO7EF&EJS{it zZ}XIHErr$;8Ot7PP??5)w$I{dauVo;58VRhZDJq;+eI`vD6Onjj#i*$@l*HoHMa&Z z-NF7u0jj@<1kP|)U8v9At`C!3aDG7&voftS8GnPz@9y8qCUD$(&tQjvFib8MiPP=q z$kHj?FruJ#RX=k!)9TUpsWUM;$->;IiYM=^_0_>&%jp&9k+r3zZKkvc2(}gK2l`QbYp(iapPM$2-nzugg7*EVgpE z$=w=eGxsKM?q4=@mA)kjiLT>FRZxI06!WEMX4j=k}A&PY~jQCX|%AVns*nS)dU!bq0!zss=U|6YdOnUIih56?l-Ox!o4 z_1`F^aImh|C#^*4EEm^35iG0S9bBp2a28-~&-t%^M{~u1%Az9(_S-$PM(Tvk`Bs7A z8VlcHZL3^2)n)AnZW~(mxa7J|?-0pzRAP3-y|9Z;NbX@gh%U!Vip)3hmIN=ikTh@y zM)}&798I%&tVrqD`v$ahal9;dB9C2!&uVDiZboz%^oxWcGXH14`N2?QZdoVO-7hrQ zT0~VICK#{l#b-RK)0XV4*My_w-&$I)P`x#F2E#3|r|h5)mT5oyauRZSYc4T)tvc*H z)9FmC1)&L_HGLZ(Mb*+bEu>4#9H_Qzytbd!P$7zZkN>bGvUhi}YcXRoc`=AuKYBK~ zdnj$Ks##BlocjshBOCcmto;?SKBMSGM<3#qe2Vt2wSS@KsYJd|6l0ykyM;C{FJbRz zXRa_cmz{6~LhgZ68EZ?E;-^Hy@>Wz7Es6)DI__{oX2e{=4kqSDim zIXg>`*Tnic<}R+d?^Sp3XO@x!m9jj~da171#Y@#2CJJo0+xZ6x3U6NX^oVR9UYPKW zu}xH-lwK{hKt`4S>esO*DDi-bVj-S+VAY|5b83v=R}3%wggNKttjH|Jt<8L53oz&a150 zrUO?1htuQ051IcX&^c?(`>?;A8v{Dmg7wXMzF=MN7JIvDrS5f8I@7!n+s)+Z0zV*! zr(Wh+EO#k>p3B;GCo+R->u7Er>q*VXJng+vgBUV)m)T}`!jE=1!u#p8$flph*ez?- z1d%xouGsFXyJWE4J`w=<7)>TJA?5VZsVu>stN?plU-sWp;cMx;Ru%tYk;m3L`5Klr zZ9Gp-?wmevrWX?4&0@oB@y=|JM{|1gEP}nIWf8-bCmO+Cz@Sh6lEJ6C5CLRmLLmKK z3?eDQ8%y;5Odg86 z;6qWQT7)$QT{o^c!?#Qy+Vl@I3Ps($-#C?x1mb)y25{AR?;$WpLLDha1h2=f>P5kK zm!TUG6_%Tyb{2c|!Msv`c^=kKZ;wD19kqJ6ML68XPXBlv*3E?0V+{3lRTs2b(M@z- zJ=Y)K@}{iyt}xFfFj<9eZe7ulcYarq>PBz(48XhqTo*tpY$;;cm z4fCs&n49=+q92zV|6G!ahK7doXl_HK%lM1$>56*`5ewH3B>ye1$pGbJs`fL9-^igj zNg#mSG5hxy|0^#6z3jPd7SPbaA<7KWjG|_XyP^pc3CTpd%x9B@0VRK z(Napq#qzmby;Za+L!teH$FAT*v|`=iO%_Q-S!!(l3_)ixTyS0ve*9{b5f_SC7T=%1 zD0Qw_vi`(mB%MO@>TEj!z-c7AhcwnSi536MU%SAI6zvl$k@GpX<8u9{>)i$eH?WoC z(By7#Encnd#jt;XX={j9Y}e1rR=RQ^G#7{#OezEnrPrfms4ChTgsWd`9scR+_jYWo3dlXlQ7k zpO~6?v|)~JLeO(O=4;VukTgWI-ei`=%gP}*#*<0bj; zA8e{hU_LyS&#zu?_7B$I>293p$X^u}eKyh_n~e5WoGw7lX_vqV!&teIVZS_80U_+I_W(wrh3U+_pYFt#sK zE)IfjRStQqXO@uvk!Ts@m)}Y$gUc7lQ*H*KUY>D0ml%Qp z_!0xhWC1x&TBk<6_4_%8gKU1AV~UG{4^NNRo0|b)C>mS?1iTz>@f$4hlq)1VWi6+H zqaj=rHD-%%kp(WkQm^a~WX5EsVYzxlS$Gh;Uqn}sll`(>tC}owKoR(}57(qgI^p>0 zfi-Ho5p2IdOKTcCt)(S$ho^94jpJ~Auqbx8I*lt#AQnXc%H3Es!oJH(;iq(3cF7BU z+P7-!wbq-kbVv-%V6pLK9@G$Y8TPa+Emmn#)X>s{c_0zqSsm%v(TR_OZQj6ixo)|hsL|Mg#~nY_i$DU9p)c&wV&C6*JvS{lSqaBId1;l#;IbMOnmy)y z?)~;nU-~v{Pz%guqqZu@TY;aQDp`b2+N@H98!4d%I2?P3)@D8iB|I2_q#zpp%8^u>;#jOPb+13{;Y zolm=CEkO$Mo7Zd2*c3&~gI?_@vIBVSCXgKJ(6sogj(PijNnpxbDP(Q7IdtmHKZ!IJ z_Zxkei~8fax(d{qY8zs|@|xAUpSC?0I48%nJVqvs#Qe4iB&AR+*(1yP!MvhpnxVYu zKL&8eDKm6zqS#IpE#@2e6Qes3iq0#QUP&)@k{tfRPFFIi#Zy~-jZC#aH7m95M1H~$rY*@89y0%h1K%1 z2;0S=SW-vyT$}Xn9n-^j(&o>_bICH6E&I2zkemDI>VgfyK?hR6u1?;=pEu){wxQ{mzt|MEPi*upYF8z3^h+of02)TwZqYTZ|S2$ zN^pSPc(qf2@KD@quYpX=Pf$xr9 zu3)w4D%U*o`_^>%P4W~kuO0{yhcYiAvYF;{oRPLn;viRHFfsUdvU8a4SMeNFc|DAZ z#a%aEYV|f=?oZZWxjtTTo1gnR2!tQ5q{gG=$|aHYAY1Gh6FA%LU6^2U#0p^CeSFJ| z^kRi>`~(A0?zjU#DR(1PV7l-k5vQ%c2u zv?-3ibp(k)1#|X|5l zMXk{oYqK&a>`QMsHyg0zsXdg==e9A3Y1Mm0*yY28N!!~QL5xOF1`j=YuvnKdB4 zusqB;|D@^=QR5nZ!0~kA_qp|mQNGTSP{F?9YH5j+3A)SOT2=`8xzFo1OgoI=1OCul zMLYMS`g%J->L=tngMG!$yC=5yX*Pn$+;n#C%^Z{gIxQxke5ds z%~(HcTgefn8xuOiAZeXjQKp3SWD35j+AIijC2>N10zD-B4XFcY|K8xln443j}jK?h|ik z8y!+`1B9%;ue|y_$?4u@0sPF&qs|>tZZ(>RzxZ!IX1rd)<1iNV>PXqXFvcN+U2lMm z$;0XHWJfbUt%}-lc(P;+?GS&+BlbdTh7pbGa=?2GIXjV0QRSGDo>ZA>d-mqNemFmi zv7soNG+A$7r;_bq)o4`pn_!c@AMbLZ3g=9tvtbe*opvrjG)J~qi-xiAL+pS*--hq2 zX+#wc5^u2GoFL_K_lbx@DEqyUy&mMV~9aqX`Dn{$x}S%F-XzX`iU)S7Dgo zEg#YZxxe!9wpwbQ<(=oBxLED-bH-*LI$N&u%HXbBI;^Q*-Ri&j<362|10E%@TfFpN zkCK@RD0Asz{Y%FG|L^*5s=+Gp3emY~1=wFlnFRdBm2S!G;r;6fx90(lFuw;=G1hM% z5)9V)#TS1ivh~lC=BETa=UqRYntv5uoO#B#EXe;{Z)(8xmLoVpn}$D+&Mv%kAAdCAVD^185W@B*9xhRGtO z1R%lQTes}FpIG`K1Wms7;0w($5Cw2|fvEnr^+C9uLuk27ORGS&rf?uJnhrqpT|QVe zz9evg)bH7>QAY1TQWaSrWC)SV2CB;rNED)?wh{2Pdwv9j>0jtyR zeBxWIdfAF=v`}ARJNZS4`csh6RI$n+Q_+Z?|1ywV>W!yQ!L@ATbh|3HKUxrRz&p@+ zVT!y=hL1HYFtDFa1uHU4f*4H|5DmvE?fW_LoMP2V4S*3o>NdYH$I4gv z?7|5~Cs1BR647aJPcUGXnhWoSrOi zs&r?elEHP?&cI^|YHHX76Yips>zz^Ih_S;E!y)N-p#V5)D|g zc8m(>F`JR&J#{hx5;xu=ygx|R@Hs4FfwVU5{q^zm=wu$2Ytd>N8$|dy7QJeIbFJkf z&Eq|C`Rnt42!yUEV2Gj==YR1#9W9u`6N;WG#{{h=Pcz7+^Ny8jsMR17HGvf)`L@|^ z(!3-AcGKcVjC#L`U@>ZDuUXC4%9ISJgt+|EHOv%%JrLYYE%wxPGg&z z)WKXX*AaH`jzkJ#Cj5ZUKo1fl;B$yVlDxknToDobk4FAV>Q970a`kwBjb5A*MU;kt zEnEZeL|)?eNnw@73oTPO0!ty(yNwN8`Fd-exLUak{z&#$lSSLpJ5}r1Lo_j+U)nIWHio^|D%k&g{(k9=ev2*NxGpGly6sTfox8oagb$r9{RfR zBEN@gO3<3I*FKf(#{N=E5COM+&)%TfU+IE_0ks*3*yWVQ?GW1+N1Kx6MEFY7TPnVp z4Cs5sm3)STL{^Ub?OB1QHag1gb&57)&20TCbP}5xmdFw;!Q&r;K}IGBa_R`Ax5<%= z`vxf7UVy~2QMb&FpOL4bE4HQ4XirV8+3bI?*rY_b7*^u;Z~d=O0$M!bT@v;`EglYt zV}$HKN&08AgEInlr6s$!`JY`fdNE{fB*nkoP+$g~B>=lZDYNy@F7X2f{CXBr$KhWE zsTV_L!ov8^F5v{^5vIGW$)Vv@6B=H7fqMVWKxAKx`Wd)=Sl1g9c8qUOpY7cC8 z2dEpYFEUf_Hr2Qu`(HP>uQrRpKvtQIZ7}yHv!wuxzU#Bg>E+k2yH18m7Tk^+6hg7+ zX@K<{^ZexW1;MQSFI4#;PBISbQCd0OhlxlOZP!#M8M$UaNoqFrd13f)kpWt!DYaUgZO=BScjvEXznN~ze~6D#g|D3WjoM19n7Exj%;U*1VKdG zyvzy-I$DptkXHE;7KWpr(1gyR@NREka z51sZ4HqJ^zLuyW+)Kv2;NMF(mYZcq=IQXEez8pIv$RFe_Hh3yVYzI)KO-FcocMsX- zi;WrpI7c9V1#rS*%}NX20tA*Ai}_k;nf1>MX#T9<8C9!;W9d}n*{zm}`w&#Mp?s7y zzKB*D^c!xpQR=@&#ptB?vyiYLa5Z<}Io#!K<6Gy(vPT$5FJ4($P;iCDp$a=}+}t5= z-b`l%#&e3(6|I!2-}=0>TKbIJNl8JG{~6^L5x+ITqz?e=T z`62*o>vx6E+U|_#jB9Br_t)To=Tct0g4;+KO5+g^LHi{#jq0M=?EJ~|c2g;d@7{&W zYpEf<(fl4zhUgS-SLSD($O3&~j0~}*8a#M}!D!!J%@ZI7)e>-8f6SGYi3d!4a<5n< z0a*Z?2I%PinwYFRikPGG;?*9f#T=BLk+33{$6fWs5#LAX(?766yN_TX)?I22t>kK; z3_&vH!v5K z_GYFfN2#n-9wf9f=EpWhxy}m~eB2e0QoQuNzeepI8vV|6vNVC+LiOz^LeGG$-PM6g z5~odaCLgE~$j0K4Ul#-8uShMwA^8Hg>5T!S)_1X6wK1*brLc-DupoZaxR2xzAv?BT z)|N!0k;YvUJ~Xb-=k<73_9ck=3x(?+)07CsQayrQzf07skW)IF_>oZ--)$G!t6Dvj z|0-O+U#myj(fv`(_Ms?h@LrsudS$PAf69Z>Y|33HoL{b2hGR0$_0p1EYzXjY6RcWi zGqd1C&ye{pyc>gHS}&(~Xqo}YF!1?NK3CcV(-EK??T%Af7o{?A)Qs?0m38j8j%71% z6@weXmk)b(N2b|cPGU}k3#Ic?2ZqDovzu3|3|BK$kL)|+B(GQ9YUOI~}mLjE_^U)9;nPJ*Z*>R}t(X-Lc+*8pvo9)X+P``w=E5ZGnvU?nmMj|Mwpk8AppeQaO^; z{TTfTWiXszVhJg#J85;(d`~ya8@2O$Ma9f#=}bC>45A+2i4rA;GXydky>@c~&f1ty;s@{aWcVrYL>_!@gRiF6tW&(CUKw&NLg+|=Vi$}HA%V_Pce!SR4 zpQP)p(Cap6_a^0*^8BOm>rr}Ugsh}n=IIY?aF9edQ7npE@@xZ@Nn~2_EfEuI#hUHh zYI}X^G-cx`2$1RPnVR?Kojw+3E2{-6cUUa9jx)teHI#nJO0Kq;m%sJ8+cgDvcU}c% z`@}Ho^)^|t2^y&m4InrV`7tKykD1e`u1e-0DKD=L$zZt1K{vowXB>PYs!(*t(q6Pzy(W z6gubs22Wq_NLst~3KX<*JYTnrbNHqpQ#9fzArSFcemZfl8>2DC?P9dc)O4PEGs3xX z;bQ>NYYUv3JT{heX1#BC?>oV)PgYQ6zesN81w0KTu?7xycz<4hSeUidbNstX5SHbLz>;uTAI;?U=}4mfizPe zp5VC}uNFUM@pRdttDEUt#93nAMH0^UTuv)hh{C%fhCKpiKhb8P&w-_@YKSINnAuRe zuh=&!X;b%y|EO=xIk3$nqKF>pNRJX!cS;WJLL+kEiYMpK0qVV91>vb`+y}X3wY)z*e|U$en>iJcNrS4T5hSi=NIhh3iHSbgDNJy1L(YcHQ_aq8=X;d)qJg z{=88kI0wbjJdzcnV!IJ*9uV4_qrj<6Ng6kz#cT6n=;&Sex%xuGBhTi=@G{QS?Py!o zl8kkU2oJ2sU@{v(e%_)Ab*PuSl1R%fuDdSO z_eSR3Ejq6w0%S(k56nngw=%iXNubipj5jhwN@}P!QHM@hqKIE=AyU{MjS2AHiHWWM z3ElciGXA?66I)2d)BpYw|d~xS*G0q(Z&71 zq+S>a5jh-?!s`;~(jIxXLj1?W$%J~s5GsRJ=eCoS#SPlrXsFFieL8gGIxD!IuUddo zng-8!ff=ERABq;-ju-1c|G1e_JMO!?)GE-^DN-tXbGiSVQ7JFR89Ab1m5Pv;1}qh` z3vs_8-^+FaoS~NcBR604dz#PrB-hK!BfO(Xx_tJ=eYO1AzfNm`TolK?E^&<-@VbM} zY*eQg@R|V#sj0(X_(bdfPKDv+h?62KAYXwdlGxaGJmUq8=!gv=JjVO?ic4 z31l&_xx&_wMY*0|=wh2Y$xa08BI~Zf=Y{fTT|Bka)UNHw)(Lfcp;6HT3#43((Y zq4JF>mT7*W$JWN`80=s|3EEYTYX61WqRcvd1Tgq%X%!zj(exZI^jmI)k+ zt(bZ}hOArL(3Ltewfm70{aubbFOp>KD)EnyP3mx-w3*}Brp;GzT#;EoQ z#!4;?YvH(cK7T0NsZSmLajE4#oce&d;Jdaya&#xZQ;}cJW7BadhVu7$VC*C$Iwvpg zL(r*mwAv=1Z?r?3r4l0Li_A62 zwMe@3wtpAV<*E`ONSRIS`Y+DE>-tUMRSeK0<~4+<)R$nIn&E&hp3Uo9aYCMf)=ncb zv57tUO6o;NT%Q9H_#Isc0&h1_U--zpJ3;_i^CEG4`QYmOzMcE~v=FgG>x@(s-=*~e z4>9cf+~4tO)rHRlL`#sXyo zpWB6;|ENF|G$5i8PsE`nfEb1G7|)gK8RUR&b>-x`LlcZtKfWgrQR~vDXn_Sy7>wsOK z+fYP-R@?jd51B>kTq4r|B-Mhd%%j*#?DOJBM3aS7TFehu+HQJ1h+M*$O%^7-Go27* zVP%wbb!}`u>$9D0z7>mdzW5WE!GsGU1n@I`4F4CZ%2EI^vu;tv(7!;B1Q-VIOq_r3 zd8Gn8kmH|z3nYR7;R5VEGv430k24yGOhZb#xc}tezY2N-kt4XT?!WictjT;5 z_a>h)qVIRCnkyo=ZWo>QQ+whZ-u%PS0E9Iv*zmt>30|Q7>VMf1BxFD+O#U7BKYR=r zWI#aY|MD?{VE}=GjsMH|`0>K{Ao?%kgA4$H`N@)f;{JuECqVEU6hv!B`M(!|Q*j6W zXSbBJxp@z1I!_F#iVC)HIwR{pFtF%-dxelzfgybeV=9WRArD7Mf*mfdQmoa}!|e;% z|6A;5Su93RO+arsirmgb@3h`0OE%a_*sipDs#NIa5QFy8lQT;;cM=77&%_95e|)~p_hm&+Jf`6^ygQ4HJ_R&WW{QY^?UOu zg$Pc-JQiH6t8Untg;*tw=v`<5mXFKnDD~lDT+a4X*O=}xCMG6FDB$cRM7zD5*4k`V zojTcahxGt}=oi`m5UgNjn6zw)3H%v}9tsppA+Pr#Dy=ZeS^uY28l#r5631MJpsi6c$x`!d}9>sg_TKE0<5efIwUf2E?N6zj1!`_ll;Wp`Tcb19lkr5OFq+ur<}pYlaiA7O}j}aDlAHSr$~>1IBiWk1OQRBZGDu9H<8n|1ac_z z`whR>KHkl+8yTQJpZ7_4!ukAs=lb=3s%4mKkn?266kN$g@4OZDH8eEh8vxR}3lpAw z3alhKFWav<4nGsFUFCb(_ILu+*=vZ|CtXn_z6ne$58z@O4@-G2y zd(wb>QctaE21E_(tM#4h<4;VAwD-NWQ)Q6nZ0q4(`?Cfmh95Fq_nbyOE#&@CPtre~QO;OMNe40@b7c-Lx z>^==7%0yklX6o#W%KZ<#vHLH@mpB>9%Xi(NU zTcYn(>VW(M;ge3_WXd3UQl(feRJ@{|!jOAil(AQF7hwhlrR|LSLDz`hLtSj)OY@Vj zZ&{Q)UhJ$<%v1hYIc{Cbm9W&CkP_r$ke8cV=yI?o-o{53>cG%HQ)62p5R90;`4vq1 z2LPL@Atkzpesf)^kM6JsLs%E*Tj=~sOdbD?Rc71b`Ip!!6P~qjoBhRe-Ss*J0Fq8u zcQX<)?ShBu)im@8(cK_45x>#+F!(p8L;4Xj;DK~84r(igIw{RNO)29{8=P){(PBP)2OeA0KYyMRiJKm$L2CVE=~RrANrY_@o@x^XX1E68ve4UhGyK z^PQMl)D&gro?m#*ZIJyt5K@3jc zJ_=Y=Q{_!EuE%K_#1YX7fu7Bfr)KtC#SBMFt_#-P9TN)qF0>znH;&ZD|HDJ5J_E1m z{Do&7okSVW+#7a{+~Ru_Jo68HzXG9sB)ynyp~k_5tHkT@p!w7ZiGz_B@(Kz=24hu+ zlU3HeCecKPB8!Mr2z|IkT2ihU#TzNP{cG2MH~7-dn8zY9BVUq5!g#9CM~f;W`J%8a zt7j<)4nR{;t|{8sFMI1uiVO1zkqN(oORKDz6rMMt5DCDnM*Z-Z+^%(PC@1_LtA*RO zz5w^28Q}Lc>mo+!U++|E0cB@wm9Lu<*?83;-@K$JCE4$K+kX)p)m+cwX)|g}mrz2I zk_LsErMY7x6&S0dCqZg64w6b5$%nhu%Sc>3ahu0LsnKCGY!2o^Y6<K=~KV&iBcZo3eMT zf_*Pa=>MsFVJTmz_dql3fBgIW#jyj|Zf#WuNhmx4Y_r>TV0AeHv3{91?)wiPKyrY5 zT=_HQG;!*9{=`4*g#RAq&W6ve_tAHaBAb+w`1x%n1&(#g?B`_)v@3D}A11eJSU3fG zWG0@5vd5ScW_7eEZ?Vd9cx_!3h2YEb6H4ea@Z zRB%HeS)6Cb#mf_+vtVjD{Q+|ql!PpoFWvcP}Iit zI`jg0$vel}9Q8Q_IdGLkUR5F6X=J>VZQ$)=up=v)o9Fg-iD!1*Z+2SYE*X@(+wtvN zkJA9xh#Fd3Tifc|9M;FE-}|{g&MwRE^IQmw&=){^z?5%as!k9GZw5@{?n1bt#CIzG zpL_-S_0@-4ZuTNA!4iug}Q$N`*9X|ih>tm7O z$6!7l{{fokf?~nI5g)oVrfWBP0HBTR*RxGCS+>AQ8Ut)VR({GeUUM0Ldh_267sYc| zOwWo^0KpEUqDOxfS6m57P=FRTCIO5Fvq-f4;GYWdHrZV`%();Qe6SA2&$3 z3wjIJ+ZPz^$m*!|2mm+< zY0Gn_`FN&2ct;On24L3PW6-%zw6EV50a%xKAq=4qhWayv0L&0b_FrPZO`y$CQBZiF zmzG=@&evasQa6A7x)`w8a-e!&5%QD*oC>FIwUR;t_abx{!MXogr|SJOEalg0fLgGC zc!k6JnJ#O7y};ABA{-@m_mZ-BjNx;0w4IpteY@oYsOX$mgPF&irvetI>3F2i zb0Z#KGOr1kJuWv)%d<317RsQ#rA!1F=vvnU7FlX z+tPgLG`BufQBO}2eSYGC;fI5ZZm7Pqv(;J}NH>JKcs_Lqd&kp!DkEewNk{qtXq=t` zcO6Fj#P7DLckoe?<~dXrBjyTtelrX|!)`b?Y}(5%b4mdB?!Ihb4Fbx@)3_h%KwCW! zfQ|S4;&dPqdQok|01srSZd0N@;Xvgi#IGg-qAC^?P09c43@O|ZF2;BnKsPz2jq!RZ zR1OzUcs2Tg@Ry|lzw-~oQ^3({mBrfFne`hi4h{!1aTY9RV<42GQG^MT||ko2(B!^*mZHRSymJm1s1_M;&X>rKV@?5~O9yWLwLh6Z=Mv6LX} zESAkozjb43D^6)3$yd|;KKJQ$+xxqyxX|AhZu%asaRDy;PX>=#Ze?=aR!Mw3H*yK0 zJR`SO6F%_d>N~}`LIMdmr%o&Wl3O|1X;Sv{Nd3>|G!^ zoxGBadBJqK&k;J=^MP_S-_lQ6-jRM&(f*~qdWH<{M72%IB$55yImhn_JUqaV90x?J z-2gN2&fQbzCc|Mj;AJUN&3c3JlK39~Vzr@Lu4uu>f2ySP;@~VC(k?H<>-@u0`Jv%qmq~j9iQQ7L+hMYJ#>Zz-t%OXnpUu=i z>!_by$dAlsMMMxf*-MZ2^z>MXa>@gV6APMT)Q&Zv`gs@6rTY}5 zSw!ADCYQMy^aYax1e1zM0WIN;v_)`I4V5!-4$jO3;_fknupq7+Qw5NjGJ`$(@f{>{ zAO7g0-h8WTV7E0*Biz7DD}YVj8O_@f^e}Cr z3KUzL^`!+b(5%2y5jxsvWgalAT8&`qvl zN^rJ0vl8bHA@ovA#HDm3i={r3;(r0PSM&aX@w?Q>#~0)hx9`BVb+$VXiYXuSEm!E6yKQb#&akF?(?n6ZR7|Qfeq=rsmKccZL26 ze(gMqQX+EfS8GM{cH5p#Du5gq0i@#mB0DI?L~x_m%+5NM2P1*fz4bFR3TvzxBus6x zcn93@*+Mk_+ybT%-KOP~i*r;nM&P({J7QDr0;;U^?QlH2f%;H(fYcoa z4)BB)2b_{RI3*WNQQq9*g7FAGpJJWs3A^Y~ktkiWLQ zDrv5$p)dM27m_NY#%@-bfT%5sPBe3M1TSIo<$ja>d<7XpdZXHvz;h6&Qt-%+m;7zRd0Wi$ZV_XHdM8wq0DUN6Q>97?F#n%rJF9Hu=~1UfkS04v3(%^s{ZN` zM1F?pR1PLW*u(>6-srWG{L3Ptb`unrMHV82(ZFk2xRi8$?D}gcXBU)IYt~MAI0CpJ z_@wT1yFJ=-d2zG;g84Rw#>ft@BD~64SmK+_PfPqp2NH*i)uPf{Qveg4Nz~u@w#oxk z^bs^JwN9WjU_o!Ulf-@Q)*j6$;J7&IlA4B+WRQY2RMd+cR6(+v2w8x-0}!7yXJ@T_ zG63LlPIMW=QGARfr6v}@0i2k|06KKqzMZHL+rFlVsn#4uV|GPkHjLdoBb4>Om*}b$ zJS4~@nqckb=Ity1T-mb(6tVA1cu^$i+OlQ)CVt5)E3dNMoHoJwus)demF)96n+_1g zR#}X^;@{rKqI71<)V43Dm~HeTscGCNU0q4j&Ch$uSVHAPODvV?HtSQz;UIp*OJt( zew`$mG*Yas#z(igOArE=Ix@nm-Rl=~Nvkl2#SchbOvg!axAagskWlf$t6U{rR&ULq zAJSQ~68M5QpI@SDodY847&oJLTc zjYW{-&0{D9Af?VJh56k8iDm~tKZ|DYDeN_wR;5#D-y;6h5pJTcg zB|-6=9;zTT=K^V6@w-VYtf(GgLdP&hUspB`uHf!7Q|d*^*9bTJ#YzX|R*bD7#4z+i_%JluUjUyUEK9G0)r|uYHKo~$HN%S` zQIK9oK1s$PJYdLA)&7cp_zXeU(J%7EX0p=K>ditkG#N8}2+q3#=6(B?@X>M;xmH0w zI%3CwnTMZBKr6tJQKup&tYy?lz1D9}%Gk$l8hxH{BN(^S6^Y+MGVsPCI7P%E(Ao-d z>DcNsWFpIH8TN9_0_5pIyP-udqe#|#I73=|4ks-5y$gbkVowQYes}>*T5oP*^Fsn0&AQ3tteEcFZPr3g@EA--x?!-}#s`an%RWAlN9u znRipvm16`%GUWk1OgbGj)&}PlK474|;>t~2&k?H|k)t*>Gr92;b-$1tnoUciI`jc$ zt&PD7$@>!EYad?rnzZWTC_Ri z^^iK$!?u1m?oi?E?0U&N3uT#T1_vsSrJn{q2kIUm%F#v8Z^@IKM=Tg}J7~}-LuSXb zS+w6-MKPtQ#;cVbayR6&-jAUWwfV`Ktvf*arbe5jT{H9KU-$|60Ej6xql}T5B#DCu zF;+9Wu8payBVhP?Q<}m zn_VezramecV*)DK`c@IlNd5xkp4bRf>kCC7 zGCnb!ZyLbA??9%ddWq>OMAn!0Fi0SJEK-R7JFg_nK_pW+AGo(tpWV3|?7NQ~vTAn2 zdPasfJ9i~DRqxl3gH>crY7`LLJ!$0P$rG$4La~08MF9B8PO?2(Sk{tPxk4w7_E)3N zF<4{pl*k<*>*QSQ)HfgyT9Gp8bY@d8j6J+DMn&@0w(&q?)Uu~~%&^=z#Gs-^xf!U{ zRJI}k45XSR>zRBEmh1CgveLRjW1&uv==QDwg*YkUUA#+b(!g=k)6Jwoz=|!aLN$Ae z)fLys?TX&lV1%BLl~v)F!)9f#fZK_!xaJY4GF#dXlv#va1|B_>teX~7yTMo&UouDp zi?LENMEm^efhK#7)=XkI2-xfC_{4|mv?t%n$Gm~-gZ`UgEg~2J-mlauC~^5I^JJrlM#qXElO$86p8LpO z6p(J<2IEx=s59||Orh;xbLQ?4;q`8$*cNWx2yK4ZV9^oq>EVyLvPRNGiVCiVo8y?-fLKNxIOr|n`HdE$a86JJx9)$E z8|yT14aV~_A|9e5j^8sIp`9eOkXY|}{We;psrol*kvqQZqT;K%2 zADSZLJ29ddp1vV*SVTfAq_h$(rPbCF@FX8kxxIcds?+wGlSanYU{<~Pzm;QV*9~Q& z7cJHdCVlZ)M8xqmP^4B=_{qKN<6Ad1^GN6)a=K92&eUy<_INN9^zX}=DMkj#Q_kh$#n`o=$-^P6CKkO*Z}K&!y*NjRU#rBuD~`Gd5j+>Haq%Uml#U= zMvg+HdJv84x-0vBgi2(Kv4w>aSL0VSJg8etlMd<^?{La0j_Xr$0 z_VdHhT87p$_R${>%alnhZhJ<28e|Np;MPAP^Ncu8_dT|>>B}cSJ_;f~1`?bM-F);m zbIp}uBJ*rs4xDzmc2y)1+3CH2h9}^;n#_9duKpN#J$TLcROn#COFD|a{*=i~5Hwij z-!sMJ-{#CSS(%=OP9km}=`M2T?KiYJ3*}brV59p4hNlgVjg=lT#g*P{M1Nx>ef^qC zKrF%+G+em|9nDUgSp0V#y6ZzjL(~0HayouXG%{O?(kViXn&mc|hnW;V->Tl@T;%&c z5Gg=gK`*-CyI%T^2dClvq%g5C$VB8yKJMJbQ=9vN=hsf;=d?x`0Lyq3I{MT#v+I`d z7x(iaYTU8GN@lO|lymR$!24NQ=Ds}yP%b1Pm)3T%wGZ9lH6sfVI?eDcqIg~-!z*Ig z?PxO3;PxY8v76iR64?*LIP*X?^L|br2SKO27G#5s(6MkuzMA`LQVi48FVaYeX6Iem@Y=p9cblBd_<$M}<^q7F7?Z&1t2e_hp){<+FS&dN(j;E`xN!FW0EJO9JCkB#lylM4Q=ZfT z_Z*LQ`*Psf@lk#q!Sg z1py1?qO*3Ehtw07w`ik0OopB=IsqX_ZO9y@6O>40L84%%1cn^EJV3#6%AO}+5E zCF+-*tkA39!n+D8nt42lgco)MM6S9!fGjBlSAXMDF!MFPE|{m{2gGq5x!Q$U%$=&PQd?rE8xg)c_vZOtDkxSDEO|(KT&>M IB4ZTzAMRvnYXATM literal 0 HcmV?d00001 diff --git a/docs/glossary.md b/docs/glossary.md new file mode 100644 index 0000000..c3fbd17 --- /dev/null +++ b/docs/glossary.md @@ -0,0 +1,6 @@ +## Concepts + +- **Software Image**: Computational infrastructure needed to carry out a run of a model. +- **Code repository**: Location where the code of a model resides (e.g., GitHub) +- **Model configuration executable**: Executable definition of the . The executable also has a code repository, which may be different from the target model. For example, I may have a model in a GitHub repository and maintain executables in another repository. +- **Model configuration metadata**: Metadata associated to a model configuration: input and output variables, creator, contributor, license, etc. diff --git a/docs/model_configuration/02-pre-steps.md b/docs/model_configuration/02-pre-steps.md new file mode 100644 index 0000000..d5c0979 --- /dev/null +++ b/docs/model_configuration/02-pre-steps.md @@ -0,0 +1,37 @@ +In order to use MIC effectively, your should create a GitHub and DockerHub account. We will use these accounts in MIC to help you publish your component. + +!!! warning + MIC **will not** store your credentials. + +## GitHub + +### Create account + +GitHub is a website and cloud-based service that helps developers store and manage their code, as well as track and control changes to their code. + +`MIC` creates a GitHub repository and push your code. + +### Obtain GitHub Token + +To push your code, you must generate a GitHub Token. Instructions for this can be found [here](../configure.md#GitHubCreds) + +!!! note + A documentation + +## DockerHub + +Docker Hub is a hosted repository service provided by Docker for sharing Docker images +If you don't have a Docker ID, head over to https://hub.docker.com to create one. + +`MIC` creates and pushes Docker images for you. + +### Login + +To push the image, you must login + +```bash +docker login +Login with your Docker ID to push and pull images from Docker Hub. If you don't have a Docker ID, head over to https://hub.docker.com to create one. +Username: frink +Password: +``` \ No newline at end of file diff --git a/docs/model_configuration/03-create-skeleton.md b/docs/model_configuration/03-create-skeleton.md new file mode 100644 index 0000000..28761ea --- /dev/null +++ b/docs/model_configuration/03-create-skeleton.md @@ -0,0 +1,12 @@ +### Creating the directory skeleton for your first configuration + +You must create the directories and sub directories + +```bash +$ mic encapsulate step1 +``` + +MIC has created a directory `model_name`. +In this directory, there two subdirectories: +- data: Contains your data/inputs. +- src: Contains the invocation script. diff --git a/docs/model_configuration/04-copy-your-data.md b/docs/model_configuration/04-copy-your-data.md new file mode 100644 index 0000000..3461f52 --- /dev/null +++ b/docs/model_configuration/04-copy-your-data.md @@ -0,0 +1,77 @@ +## Identify your inputs and copy them + +You must copy your inputs into the directory `data`. + +!!! warning + Your code is not an input. + +An `input` can be: + +- A file in the directory `data` is one input. +- A directory in the directory `data` is one input (MIC is going create a zip file). + +Let's suppose that you have copied the following directory and file +- GLDAS_NOAH025_M.2.1/ - This is a directory +- prepicipitation_rates.txt - This is a file + + +## Identify your parameters + +Analysts may want to explore indicators values under different initial conditions. These are expressed as adjustable parameters of models. + + +Let's suppose that you have identified two parameters: +- start_year: +- end_year + + +## Creating `config.yaml` file + +Then, you must run the command: + +```bash +$ mic encapsulate step2 --inputs_dir data/ --number-parameters 2 +or +$ mic encapsulate step2 --number-parameters 2 +``` + +This command generates `config.yaml` file. This YAML file with the information about your model configuration + +```yaml +inputs: + gldas_noaho25_m.2.1: + path: data/GLDAS_NOAH025_M.2.1/ + prepicipitation_rates: + path: data/prepicipitation_rates.txt +parameters: + parameter1: + default_value: + parameter2: + default_value: +``` + +You **must** add: + + - A *default_value* for each parameter + +You **can** edit + + - The name of the parameters and inputs (Spaces are not admitted) + +### Creating the invocation code + +Then, we must generate the MINT wrapper to run your model + +You must pass the `MIC_CONFIG_FILE` (`config.yaml`) using the option (`-f`). + + +```bash +$ mic encapsulate step3 -f config.yaml +The invocation has been created. +``` + +!!! warning + If you edit the inputs or the parameters section in the `config.yaml` file, you must re-run ` mic encapsulate step3 -f config.yaml` + + +In the next step, you are going to learn how to run your models using the MINT Wrapper diff --git a/docs/model_configuration/05-write-invocation.md b/docs/model_configuration/05-write-invocation.md new file mode 100644 index 0000000..07bb920 --- /dev/null +++ b/docs/model_configuration/05-write-invocation.md @@ -0,0 +1,61 @@ +## MINT wrapper + +You must add the invocation line of your model, + +### Types + +An invocation can be one line. + +For example, FloodSeverityIndex model. [Example](https://github.com/mintproject/MINT-WorkflowDomain/blob/master/WINGSWorkflowComponents/fsi-1.0.0/src/run#L19) + +```bash +python FloodSeverityIndex.py ./ GloFAS_FloodThreshold.nc [23,48,3,15] [2016,2017] True +``` + +Or multiple lines as the HAND model. [Example](https://github.com/mintproject/HAND-TauDEM/blob/master/hand_v2_mint_component/src/run#L27) + + +```bash +... +pitremove -z $1 -fel demfel.tif +dinfflowdir -fel demfel.tif -ang demang.tif -slp demslp.tif +d8flowdir -fel demfel.tif -p demp.tif -sd8 demsd8.tif +aread8 -p demp.tif -ad8 demad8.tif -nc +areadinf -ang demang.tif -sca demsca.tif -nc + +## Skeleton +slopearea -slp demslp.tif -sca demsca.tif -sa demsa.tif +d8flowpathextremeup -p demp.tif -sa demsa.tif -ssa demssa.tif -nc +python3 hand-thresh.py --resolution demfel.tif --output demthresh.txt +threshold -ssa demssa.tif -src demsrc.tif -thresh 500 + +streamnet -fel demfel.tif -p demp.tif -ad8 demad8.tif -src demsrc.tif -ord demord.tif -tree demtree.dat -coord demcoord.dat -net demnet.shp -w demw.tif -sw + +connectdown -p demp.tif -ad8 demad8.tif -w demw.tif -o outlets.shp -od movedoutlets.shp + +python3 hand-heads.py --network demnet.shp --output dangles.shp +python3 hand-weights.py --shapefile dangles.shp --template demfel.tif --output demwg.tif +``` + +What is the best option? That's is your decision. We provide flexibility. + + + + +### Adding your invocation line + +!!! info + The language of the run file is bash. + + +1. Open the file `src/run` +2. Add the invocation line(s) after the comment `# WRITE THE COMMAND LINE INVOCATION HERE.` + + +``` +# WRITE THE COMMAND LINE INVOCATION HERE +python FloodSeverityIndex.py ./ GloFAS_FloodThreshold.nc [23,48,3,15] [2016,2017] True +``` + + +On the next page, we are going to learn how to pass the parameters to your model. \ No newline at end of file diff --git a/docs/model_configuration/06-pass-parameters.md b/docs/model_configuration/06-pass-parameters.md new file mode 100644 index 0000000..d505be4 --- /dev/null +++ b/docs/model_configuration/06-pass-parameters.md @@ -0,0 +1,92 @@ +# Passing the parameter to your model + +Analysts may want to explore indicators values under different initial conditions. These are expressed as adjustable parameters and input variables of models. + +We identified two ways to pass the parameters to your model. To explain this, we are going to use example `config.yaml` + + +In this case, we have two basic parameters: + +- start_date +- end_date + +```yaml +... +parameters: + start_date: + default_value: 2010 + end_date: + default_value: 2012 +``` + + +## Arguments + +Some models read the parameters from the command line. For example, the invocation line for cycles is: + +```bash +python3 cycles-wrapper.py --start-year 2010 --end-year 2012 +``` + +Then, you must replace the value by the variable in the invocation line +```bash +python3 cycles-wrapper.py --start-year ${start_date} --end-year ${end_date} +``` + +## Configuration file + +Some models read the parameters from a configuration file. + +For example, the `SWAT+` model has the following invocation line. + +```bash +swatplus +``` + +SWAT uses configuration files to pass the parameters. To manipulate the simulation dates, SWAT has a file named `time.sim` + +``` + DAY_START YRC_START DAY_END YRC_END STEP + 0 2001 0 2003 0 +``` + +Then, we must open the file and replace the values with variables +```bash + DAY_START YRC_START DAY_END YRC_END STEP + 0 ${start_date} 0 ${end_date} 0 +``` + +And add the file as configuration file of the model. + +```bash +mic encapsulate step4 -f config.yaml [configuration_files]... +``` + +In the example, we must run +``` +mic encapsulate step4 -f config.yaml`src/time.sim` +``` + +And the `config.yaml` has been updated + +``` +```yaml +inputs: + gldas_noaho25_m.2.1 + path: data/GLDAS_NOAH025_M.2.1/ + pre + path: data/prepicipitation_rates.txt +parameters: + - name: start_date + default_value: 2010 + - name: end_date + default_value: 2012 +config_files + - src/time.sim +``` + +!!! warning + If you edit the inputs or the parameters section in the `config.yaml` file, you must re-run `mic model_configuration init config.yaml` + +!!! info + We are using a standard template language JINJA \ No newline at end of file diff --git a/docs/model_configuration/07-validate.md b/docs/model_configuration/07-validate.md new file mode 100644 index 0000000..1a40cfe --- /dev/null +++ b/docs/model_configuration/07-validate.md @@ -0,0 +1,26 @@ +It's time to run and validate your component. + +If you don't want to test the Docker Image, add the option `--no-docker` + +```bash +$ mic model_configuration validate hello_world --no-docker +``` + +The validation process is going to validate: +- Pass the parameters to your model +- The execution on your machine +- The creation of the Docker Image +- The execution on your machine using Docker + +```bash +$ mic model_configuration validate hello_world/ +[OK] Parameters +[OK] Execution without Docker +[OK] Extraction of dependencies +[OK] Build Docker Image +[OK] Execution using Docker + +Created: hello_world/hello_world.zip +``` + +The result is going to be a zip file. \ No newline at end of file diff --git a/docs/model_configuration/08-outputs.md b/docs/model_configuration/08-outputs.md new file mode 100644 index 0000000..14dd88e --- /dev/null +++ b/docs/model_configuration/08-outputs.md @@ -0,0 +1,43 @@ +## Identify the output files + +Now, we must identify the outputs of your model. + +!!! question + Why is it important? + You can add metadata about the outputs such as Units, formats, etc. + +The output of your models are in the directory src/ + +Let's suppose that the output files are the file `file.txt` and the directory `src/images` + +You must run: + +```bash +$ mic modelconfiguration outputs src/file.txt src/images +``` + +Then, the config file `config.yaml` has been updated + +```yaml +inputs: + - name: gldas_noaho25_m.2.1 + path: data/GLDAS_NOAH025_M.2.1/ + - name: + path: data/prepicipitation_rates.txt +parameters: + - name: start_date + default_value: 2010 + - name: end_date + default_value: 2012 +outputs: + - name: output_file + path: src/file.txt + - name: output_images + path: src/images +``` + +Don't edit the outputs section manually! + +!!! warning + - If you edit the inputs or the parameters section in the `config.yaml` file. You must return to the [steop] + diff --git a/docs/model_configuration/09-publish.md b/docs/model_configuration/09-publish.md new file mode 100644 index 0000000..9302051 --- /dev/null +++ b/docs/model_configuration/09-publish.md @@ -0,0 +1,10 @@ +It's time to upload your code, image and model configuration. + +``` +$ mic model_configuration push +Pushing image +Pushing Git repository +Pushing Model Configuration + +URL Model Configuration +``` \ No newline at end of file diff --git a/docs/model_configuration/docker.md b/docs/model_configuration/docker.md new file mode 100644 index 0000000..32c6af8 --- /dev/null +++ b/docs/model_configuration/docker.md @@ -0,0 +1 @@ +## \ No newline at end of file diff --git a/docs/usage-add-configuration.md b/docs/usage-add-configuration.md new file mode 100644 index 0000000..a8debb1 --- /dev/null +++ b/docs/usage-add-configuration.md @@ -0,0 +1,3 @@ + + +- \ No newline at end of file diff --git a/mkdocs.yml b/mkdocs.yml index 21a06c5..63803fe 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -4,7 +4,17 @@ nav: - Available commands: usage.md - Configure: configure.md - Add model: usage-add-model.md + - 'Add modelconfiguration': + - 'Before you start...': 'model_configuration/02-pre-steps.md' + - 'Getting started': 'model_configuration/03-create-skeleton.md' + - 'Inputs and parameters': 'model_configuration/04-copy-your-data.md' + - 'Write the invocation lines': 'model_configuration/05-write-invocation.md' + - 'Pass the parameters': 'model_configuration/06-pass-parameters.md' + - 'Run and Validate': 'model_configuration/07-validate.md' + - 'Identify the outputs': 'model_configuration/08-outputs.md' + - 'Publish': 'model_configuration/09-publish.md' - FAQ: faq.md + - Glossary: glossary.md theme: name: material diff --git a/pytest.ini b/pytest.ini new file mode 100644 index 0000000..2c0bc0a --- /dev/null +++ b/pytest.ini @@ -0,0 +1,6 @@ +[pytest] +testpaths = src/mic/tests +norecursedirs=dist build .tox scripts src/mic/resources +addopts = + --basetemp=mydir +# src/mic/tests diff --git a/setup.py b/setup.py index 8a8b39f..02a9119 100644 --- a/setup.py +++ b/setup.py @@ -7,8 +7,13 @@ "jsonschema>=3.0.0", "semver>=2.8.1", "requests", + "tabulate>=0.8.1", + "Jinja2>=2.11.2", + "PyYAML>=5.3.1", "modelcatalog-api==2.5.0", - "tabulate>=0.8.1" + "dame-cli>=5.0.0", + "pygit2>=1.2.1", + "PyGithub>=1.43.5" ] diff --git a/src/mic/__main__.py b/src/mic/__main__.py index 169792c..bb5b3a1 100644 --- a/src/mic/__main__.py +++ b/src/mic/__main__.py @@ -1,15 +1,25 @@ import sys from pathlib import Path -import click import mic import semver -from mic.credentials import configure_credentials +from dame.utils import obtain_id from mic import _utils, file +from mic.cli_docs import * +from mic.component.executor import execute_using_docker +from mic.component.initialization import create_directory, render_run_sh, render_io_sh, render_output, \ + render_dockerfile, render_gitignore +from mic.config_yaml import fill_config_file_yaml, get_numbers_inputs_parameters, get_inputs_parameters, \ + add_configuration_files, create_config_file_yaml, get_spec, write_step, write_spec +from mic.constants import * +from mic.credentials import configure_credentials, print_list_credentials +from mic.drawer import print_choices +from mic.model_catalog_utils import get_label_from_response +from mic.publisher.docker import publish_docker +from mic.publisher.github import create_local_repo_and_commit, push +from mic.publisher.model_catalog import create_model_catalog_resource, publish_model_configuration from mic.resources.model import create as create_model -from mic.resources.model_configuration import create as model_configuration_create - -from modelcatalog import Configuration +from modelcatalog import Configuration, DatasetSpecification, Parameter @click.group() @@ -27,12 +37,13 @@ def cli(verbose): ) -@cli.command(short_help="Show mic version.") +@cli.command(short_help="Show mic version") def version(debug=False): click.echo(f"{Path(sys.argv[0]).name} v{mic.__version__}") -@cli.command(help="Configure your credentials to access the Model Catalog API ") +@cli.command(short_help="Configure credentials", help="Configure your credentials to access the Model Catalog API, " + "GitHub and Docker features") @click.option( "--profile", "-p", @@ -44,15 +55,43 @@ def version(debug=False): @click.option('--server', prompt='Model Catalog API', help='The Model Catalog API', required=True, default=Configuration().host, show_default=True) @click.option('--username', prompt='Username', - help='Your email.', required=True, default="mint@isi.edu", show_default=True) + help='Your email', required=True, default="mint@isi.edu", show_default=True) @click.option('--password', prompt="Password", required=True, hide_input=True, help="Your password") -def configure(server, username, password, profile="default"): +@click.option('--name', prompt='Name', help='Your name', required=True) +@click.option('--git_username', prompt='GitHub Username', help='Your GitHub Username', required=True) +@click.option('--git_token', prompt='GitHub API token', help='Your GitHub API token', required=True, hide_input=False) +@click.option('--dockerhub_username', prompt='Docker Username', help='Your Docker Username') +def credentials(server, username, password, git_username, git_token, name, dockerhub_username, profile="default"): try: - configure_credentials(server, username, password, profile) + email = username + configure_credentials(server, username, password, git_username, git_token, name, email, dockerhub_username, + profile) except Exception as e: click.secho("Unable to create configuration file", fg="red") + +@cli.command(short_help="List credentials profiles", + help="List credential parameters for mic profiles. Lists all profile credentials if no profile given") +@click.option( + "--profile", + "-p", + envvar="MINT_PROFILE", + type=str, + default=None, + metavar="", + help="specify a specific profile to list" +) +@click.option( + "--short", + "-s", + is_flag=True, + help="Only show a list of profiles, not their contents" +) +def list_credentials(profile=None, short=False): + print_list_credentials(profile, short) + + @cli.group() def model(): """Command to create and edit Models""" @@ -95,11 +134,205 @@ def load(filename, profile): @cli.group() -def modelconfiguration(): - """Command to create and edit ModelConfigurations""" +def encapsulate(): + """Command to encapsulate your Model Configuration""" + + +@encapsulate.command(short_help="Create directories and subdirectories") +@click.argument( + "model_configuration_name", + type=click.Path(exists=False, dir_okay=True, file_okay=False, resolve_path=True), + required=True +) +def step1(model_configuration_name): + """ + Create the directories and subdirectories. + mic encapsulate step1 -@modelconfiguration.command(short_help="Create a modelconfiguration") + The argument: `model_configuration_name` is the name of your model configuration + """ + try: + model_dir_path = create_directory(Path('.'), model_configuration_name) + except Exception as e: + click.secho("Error: {} could not be created".format(model_configuration_name), fg="red") + exit(1) + + render_gitignore(model_dir_path) + create_config_file_yaml(model_dir_path) + create_local_repo_and_commit(model_dir_path) + click.echo("MIC has created the directories") + click.secho( + "You must add your data (files or directories) into the directory: {}".format(model_dir_path / DATA_DIR), + fg='green') + + +@encapsulate.command(short_help="Pass the inputs and parameters for your Model Configuration") +@click.option( + "-p", + "--parameters", + type=int, + required=True, + default=0 +) +@click.option( + "-f", + "--mic_config_file", + type=click.Path(exists=True, dir_okay=False, file_okay=True, resolve_path=True), + default=CONFIG_YAML_NAME +) +def step2(mic_config_file, parameters): + """ + Fill the MIC configuration file with the information about the parameters and inputs + + mic encapsulate step2 -f -p + + MIC is going to detect: + - the inputs (files and directory) and add them in the MIC configuration file. + - the parameters and add them in the configuration file + + """ + inputs_dir = Path(mic_config_file).parent / DATA_DIRECTORY_NAME + if not inputs_dir.exists(): + exit(1) + fill_config_file_yaml(Path(mic_config_file), inputs_dir, parameters) + + +@encapsulate.command(short_help="Create MINT wrapper using the " + CONFIG_YAML_NAME) +@click.option( + "-f", + "--mic_config_file", + type=click.Path(exists=True, dir_okay=False, file_okay=True, resolve_path=True), + default=CONFIG_YAML_NAME +) +def step3(mic_config_file): + """ + Create MINT wrapper using the mic.yaml + + - You must pass the MIC_CONFIG_FILE (mic.yaml) using the option (-f). + + mic encapsulate step3 -f + """ + if not Path(mic_config_file).exists(): + click.secho("Error: {} doesn't exists".format(mic_config_file), fg="red") + exit(1) + config_path = Path(mic_config_file) + model_directory_path = config_path.parent + inputs, parameters, outputs, configs = get_inputs_parameters(config_path) + number_inputs, number_parameters, number_outputs = get_numbers_inputs_parameters(config_path) + run_path = render_run_sh(model_directory_path, inputs, parameters, number_inputs, number_parameters) + render_io_sh(model_directory_path, inputs, parameters, configs) + render_output(model_directory_path, [], False) + spec = get_spec(config_path) + write_step(config_path, spec, 3) + click.secho("The MINT Wrapper has created: {}".format(run_path)) + + +@encapsulate.command(short_help="If the configuration has config files, select them") +@click.argument( + "configuration_files", + type=click.Path(exists=True, dir_okay=False, file_okay=True, resolve_path=True), + required=True, + nargs=-1 +) +@click.option( + "-f", + "--mic_config_file", + type=click.Path(exists=True, dir_okay=False, file_okay=True, resolve_path=True), + default=CONFIG_YAML_NAME +) +def step4(mic_config_file, configuration_files): + """ + THIS IS STEP IS OPTIONAL + + Select the inputs files that are configuration files + + - You must pass the MIC_CONFIG_FILE (mic.yaml) using the option (-f). + + - And the files as arguments + + mic encapsulate step4 -f [configuration_files]... + + For example, + mic encapsulate step4 -f mic.yaml data/example_dir/file1.txt data/file2.txt + """ + config_path = Path(mic_config_file) + if not config_path.exists(): + exit(1) + add_configuration_files(config_path, configuration_files) + model_directory_path = config_path.parent + inputs, parameters, outputs, configs = get_inputs_parameters(config_path) + number_inputs, number_parameters, number_outputs = get_numbers_inputs_parameters(config_path) + render_run_sh(model_directory_path, inputs, parameters, number_inputs, number_parameters) + render_io_sh(model_directory_path, inputs, parameters, configs) + render_output(model_directory_path, [], False) + spec = get_spec(config_path) + write_step(config_path, spec, 4) + + +@encapsulate.command(short_help="Optional - Run your model with your computational environment.") +@click.option( + "-f", + "--mic_config_file", + type=click.Path(exists=True, dir_okay=False, file_okay=True, resolve_path=True), + default=CONFIG_YAML_NAME +) +def step5(mic_config_file): + """ + Editing the MIC Wrapper and building your environment + For example, + + mic encapsulate step5 -f + """ + mic_config_path = Path(mic_config_file) + model_dir = mic_config_path.parent + src_dir_path = model_dir / SRC_DIR + if not mic_config_path.exists(): + exit(1) + # framework = detect_framework(src_dir_path) + # if framework is None: + # click.secho("We need information about the language, tool or framework used by the model") + # click.secho("This information allows to select the correct Docker Image") + # click.secho("By the default, you can select the option {}".format(Framework.GENERIC)) + # framework = click.prompt("Select a option ".format(Framework), + # show_choices=True, + # type=click.Choice(Framework, case_sensitive=False), + # value_proc=handle + # ) + framework = Framework.GENERIC + if framework == Framework.GENERIC: + bin_dir = model_dir / DOCKER_DIR / "bin" + bin_dir.mkdir(exist_ok=True) + dockerfile = render_dockerfile(model_dir, framework) + click.secho("Dockerfile has been created: {}".format(dockerfile)) + spec = get_spec(mic_config_path) + + +@encapsulate.command(short_help="Build and run the Docker Image") +@click.option( + "-f", + "--mic_config_file", + type=click.Path(exists=True, dir_okay=False, file_okay=True, resolve_path=True), + default=CONFIG_YAML_NAME +) +def step6(mic_config_file): + """ + Build and run the Docker image + + mic encapsulate step6 -f + """ + mic_config_path = Path(mic_config_file) + execute_using_docker(Path(mic_config_file)) + write_spec(mic_config_path, STEP_KEY, 7) + + +@encapsulate.command(short_help="Publish your code") +@click.option( + "-f", + "--mic_config_file", + type=click.Path(exists=True, dir_okay=False, file_okay=True, resolve_path=True), + default=CONFIG_YAML_NAME +) @click.option( "--profile", "-p", @@ -108,7 +341,71 @@ def modelconfiguration(): default="default", metavar="", ) -def add(profile): - from mic.resources.software_version import SoftwareVersionCli - model_configuration_create(profile=profile, parent=SoftwareVersionCli) - click.secho(f"Success", fg="green") +def step7(mic_config_file, profile): + """ + Publish your code and MIC wrapper on GitHub and the Docker Image on DockerHub + + mic encapsulate step7 -f + """ + info_step8() + mic_config_path = Path(mic_config_file) + model_dir = mic_config_path.parent + click.secho("Deleting the executions") + push(model_dir, mic_config_path, profile) + publish_docker(mic_config_path, profile) + write_spec(mic_config_path, STEP_KEY, 8) + + +@encapsulate.command(short_help="Publish your model configuration") +@click.option( + "-f", + "--mic_config_file", + type=click.Path(exists=True, dir_okay=False, file_okay=True, resolve_path=True), + default=CONFIG_YAML_NAME +) +@click.option( + "--profile", + "-p", + envvar="MINT_PROFILE", + type=str, + default="default", + metavar="", +) +def step8(mic_config_file, profile): + mic_config_path = Path(mic_config_file) + model_configuration = create_model_catalog_resource(Path(mic_config_file), allow_local_path=False) + api_response_model, api_response_mc = publish_model_configuration(model_configuration, profile) + click.echo("You can run or see the details using DAME. More info at https://dame-cli.readthedocs.io/en/latest/") + click.echo("For example, you can run it using:\ndame run {}".format(obtain_id(api_response_mc.id))) + write_spec(mic_config_path, STEP_KEY, 9) + +@encapsulate.command(short_help="Show status") +@click.option( + "-f", + "--mic_config_file", + type=click.Path(exists=True, dir_okay=False, file_okay=True, resolve_path=True), + default=CONFIG_YAML_NAME +) +def status(mic_config_file): + mic_config_path = Path(mic_config_file) + spec = get_spec(mic_config_path) + click.secho("Step {} of {}".format(spec[STEP_KEY], TOTAL_STEPS)) + + +def prepare_inputs_outputs_parameters(inputs, model_configuration, name): + _inputs = [] + _outputs = [] + _parameters = [] + for i in range(0, inputs): + _inputs.append(DatasetSpecification(label="Input {}".format(i + 1), position=i + 1)) + for i in range(0, inputs): + _outputs.append(DatasetSpecification(label="Output {}".format(i + 1), position=i + 1)) + for i in range(0, inputs): + _parameters.append(Parameter(label="Parameter {}".format(i + 1), position=i + 1)) + if _inputs: + model_configuration.has_input = _inputs + if _outputs: + model_configuration.has_output = _outputs + if _parameters: + model_configuration.has_parameter = _parameters + model_configuration.label = name diff --git a/src/mic/_makeyaml.py b/src/mic/_makeyaml.py new file mode 100644 index 0000000..84c7779 --- /dev/null +++ b/src/mic/_makeyaml.py @@ -0,0 +1,90 @@ +import logging +import click +import yaml + +from mic import _schema + +logger = logging.getLogger() +schemaDefinitions = _schema.get_schema()["definitions"] +error_log = "" + + +def make_yaml(config_yaml_path): + if config_yaml_path.exists(): + response = click.confirm("\"{}\" already exists. Do you want to overwrite it?", default=False) + if not response: + click.echo("Aborting YAML Generation") + exit(0) + yaml_outline = write_properties(_schema.get_schema()["properties"]) + with open(config_yaml_path, "w+") as f: + yaml.dump(yaml_outline, f, sort_keys=False) + + +def write_properties(prop): + dict = {} + + for i in prop: + # print(i + ": " + str(type((prop[i])["type"]))) + curr = prop[i] + + try: + # if there is only one type for the current property + if type(curr["type"]) is str: + if curr["type"] == "string": + if i == "schemaVersion": + dict[i] = _schema.get_schema_version() + else: + dict[i] = "" + elif curr["type"] == "array": + ci = curr["items"] + if "$ref" in list(ci.keys()): + ref = ci["$ref"] + ref = ref.split('/') + ref = ref[-1] + dict[i] = [write_properties((schemaDefinitions[ref])["properties"])] + elif "type" in list(ci.keys()): + dict[i] = [] + else: + logger.warning("unknown type fount in " + i) + + elif curr["type"] == "integer": + dict[i] = 0 + elif curr["type"] == "float": + dict[i] = 0.0 + elif curr["type"] == "boolean": + dict[i] = False + elif curr["type"] == "object": + try: + dict[i] = write_properties(curr["properties"]) + except KeyError as ke: + logger.info("In object: \"" + i + "\" couldn't find type: " + str(ke) + "making empty object") + dict[i] = {} + # if there are multiple types for the given property. Favoring in order: objects, arrays, everything else + elif type(curr["type"]) is list: + if "object" in curr["type"]: + try: + dict[i] = write_properties(curr["properties"]) + except KeyError as ke: + logger.info("In object: \"" + i + "\" couldn't find type " + str(ke) + " making empty object") + dict[i] = {} + elif "array" in curr["type"]: + dict[i] = [] + else: + dict[i] = "" + + except KeyError as err: + logger.warning("\"KeyError\" in \"" + i + "\" no \"" + str(err) + "\"") + + return dict + + +def _main(): + make_yaml() + + +if __name__ == "__main__": + try: + _main() + except Exception as e: + logger.exception(e) + diff --git a/src/mic/_schema.py b/src/mic/_schema.py new file mode 100644 index 0000000..54e76b4 --- /dev/null +++ b/src/mic/_schema.py @@ -0,0 +1,66 @@ +# -*- coding: utf-8 -*- + +import logging + +from jsonschema import Draft7Validator + + +schemaVersion = "0.0.1" + +schema = { + "type": "object", + "required": ["inputs", "parameters", "outputs"], + "properties": { + "schemaVersion": {"type": "string"}, + "inputs": {"type": "object", "$ref": "#/definitions/data_file"}, + "outputs": {"type": "object", "$ref": "#/definitions/data_file"}, + "parameters": {"type": "object", "$ref": "#/definitions/parameters"}, + }, + + "definitions": { + "data_file": { + "type": ["object"], + "required": ["name", "path"], + "properties": { + "name": {"type": "string"}, + "path": {"type": "string"}, + }, + }, + "parameters": { + "type": ["object"], + "required": ["name", "default_value"], + "properties": { + "name": {"type": "string"}, + "default_value": {"type": ["string", "number", "bool"]}, + }, + }, + }, + +} +# Missing extension for variables of each input and variable of each output + +v = Draft7Validator(schema) + + +def get_schema(): + return schema + + +def get_schema_version(): + return schemaVersion + + +def _msg(e): + """Generate a user friendly error message.""" + return e.message + + +def check_package_spec(spec): + """Check package specification.""" + err = [] + for e in v.iter_errors(spec): + err.append(_msg(e)) + logging.error(_msg(e)) + + if err: + raise ValueError("Invalid component specification.") diff --git a/src/mic/_utils.py b/src/mic/_utils.py index 2cf30e0..ed5708b 100644 --- a/src/mic/_utils.py +++ b/src/mic/_utils.py @@ -4,16 +4,42 @@ import uuid import click import requests +import validators from mic._mappings import Metadata_types MODEL_ID_URI = "https://w3id.org/okn/i/mint/" __DEFAULT_MINT_API_CREDENTIALS_FILE__ = "~/.mint/credentials" +def path_walk(top, topdown = False, followlinks = False): + """ + See Python docs for os.walk, exact same behavior but it yields Path() instances instead + """ + names = list(top.iterdir()) + + dirs = (node for node in names if node.is_dir() is True) + nondirs =(node for node in names if node.is_dir() is False) + + if topdown: + yield top, dirs, nondirs + + for name in dirs: + if followlinks or name.is_symlink() is False: + for x in path_walk(name, topdown, followlinks): + yield x + + if topdown is not True: + yield top, dirs, nondirs + + def generate_new_uri(): return "{}{}".format(MODEL_ID_URI, str(uuid.uuid4())) +def obtain_id(url): + if validators.url(url): + return url.split('/')[-1] + def first_line_new(resource, i=""): click.echo("======= {} ======".format(resource)) click.echo("The actual values are:") diff --git a/src/mic/cli_docs.py b/src/mic/cli_docs.py new file mode 100644 index 0000000..3a18b9c --- /dev/null +++ b/src/mic/cli_docs.py @@ -0,0 +1,5 @@ +import click + + +def info_step8(): + click.echo("This step publishes your code, DockerImage and ModelConfiguration") \ No newline at end of file diff --git a/src/mic/component/__init__.py b/src/mic/component/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/mic/component/dame.py b/src/mic/component/dame.py new file mode 100644 index 0000000..e69de29 diff --git a/src/mic/component/executor.py b/src/mic/component/executor.py new file mode 100644 index 0000000..1f787ad --- /dev/null +++ b/src/mic/component/executor.py @@ -0,0 +1,202 @@ +import logging +import os +import shutil +import subprocess +from datetime import datetime +from pathlib import Path + +import click +import docker +from dame.executor import build_parameter, build_output +from mic.component.initialization import render_output +from mic.config_yaml import get_inputs_parameters, write_spec, add_outputs +from mic.constants import SRC_DIR, EXECUTIONS_DIR, DOCKER_DIR, DOCKER_KEY, LAST_EXECUTION_DIR +from mic.publisher.model_catalog import create_model_catalog_resource + + +def copy_file(input_path: Path, src_dir_path: Path): + return shutil.copyfile(input_path, src_dir_path / input_path.name) + + +def compress_directory(mint_config_file: Path): + pass + + +def _copy_directory(src: Path, dest: Path) -> Path: + return shutil.copytree(src, dest) + + +def copy_inputs(mint_config_file: Path, src_dir_path: Path): + model_path = mint_config_file.parent + inputs, parameters, _, _ = get_inputs_parameters(mint_config_file) + for _, item in inputs.items(): + input_path = model_path / item['path'] + is_directory = True if input_path.is_dir() else False + try: + if is_directory: + shutil.copytree(input_path, src_dir_path / input_path.name) + else: + shutil.copy(input_path, src_dir_path / input_path.name) + click.secho("Added: {} into the execution directory".format(input_path.name), fg="green") + except OSError as e: + click.secho("Failed: Error message {}".format(e), fg="red") + except Exception as e: + click.secho("Failed: Error message {}".format(e), fg="red") + click.secho("The execution directory is available {}".format(src_dir_path), fg="green") + + +def create_execution_directory(mint_config_file: Path, model_path: Path): + from datetime import datetime + execution_name = datetime.now().strftime("%m_%d_%H_%M_%S") + execution_dir = model_path / EXECUTIONS_DIR / execution_name + execution_dir.mkdir(parents=True) + src_executions_dir = execution_dir / SRC_DIR + _copy_directory(model_path / SRC_DIR, src_executions_dir) + copy_inputs(mint_config_file, src_executions_dir) + return src_executions_dir + + +def clean_execution_directory(model_path: Path): + execution_dir = model_path / EXECUTIONS_DIR + shutil.rmtree(execution_dir) + + +def run_execution(line, execution_dir): + proc = subprocess.Popen(line.split(' '), cwd=execution_dir) + proc.wait() + return proc.returncode + + +def execute(mint_config_file: Path): + model_path = mint_config_file.parent + execution_dir = create_execution_directory(mint_config_file, model_path) + resource = create_model_catalog_resource(mint_config_file) + try: + line = get_command_line(resource) + except: + logging.error("Unable to cmd_line", exc_info=True) + click.secho("Running\n{}".format(line)) + if run_execution(line, execution_dir) == 0: + click.secho("Success", fg="green") + else: + click.secho("Failed", fg="red") + + +def build_docker(docker_path: Path, name: str): + client = docker.from_env() + image, logs = client.images.build(path=str(docker_path), tag="{}".format(name), nocache=True) + for chunk in logs: + print(chunk) + return image.tags[0] + # return docker_image_name + + +def execute_using_docker(mint_config_file: Path): + model_path = mint_config_file.parent + name = model_path.name + docker_path = model_path / DOCKER_DIR + image = build_docker(docker_path, name) + now = datetime.now().timestamp() + + src_dir = create_execution_directory(mint_config_file, model_path) + try: + resource = create_model_catalog_resource(mint_config_file) + except ValueError: + pass + + docker_run(image, resource, src_dir) + detect_news_file(src_dir, mint_config_file, now) + write_spec(mint_config_file, LAST_EXECUTION_DIR, str(src_dir.absolute())) + return src_dir + + +def docker_run(image, resource, src_dir): + mint_volumes = {str(src_dir.absolute()): {'bind': '/tmp/mint', 'mode': 'rw'}} + try: + line = get_command_line(resource) + except: + logging.error("Unable to cmd_line", exc_info=True) + click.secho("Running \n {}".format(line), fg="green") + try: + client = docker.from_env() + res = client.containers.run(command=line, + image=image, + volumes=mint_volumes, + working_dir='/tmp/mint', + detach=True, + stream=True, + remove=True + ) + for chunk in res.logs(stream=True): + print(chunk) + click.secho("Success", fg="green") + + except Exception as e: + click.secho("Failed", fg="red") + logging.error(e, exc_info=True) + + +def compress_file(detected_files): + return click.confirm("Do you want to create one zip files with the files?", default=False) + + +def detect_news_file(src_directory: Path, mint_config_file: Path, time: datetime): + """ + Get the files by a modification timestamp. + :param src_directory: The src execution dir + :type src_directory: Path + :param mint_config_file: The mic configuration file + :type mint_config_file: Path + :param time: Execution time + :type time: datetime + """ + model_name = mint_config_file.parent.name + files_list = [] + for root, _, filenames in os.walk(src_directory, topdown=True): + for filename in filenames: + filepath = os.path.join(os.path.abspath(root), filename) + created = os.path.getmtime(Path(filepath)) + modified = os.path.getmtime(Path(filepath)) + if time < created or time < modified: + files_list.append(Path(filepath).relative_to(src_directory)) + if files_list: + model_dir = mint_config_file.parent + click.secho("The model has generated the following files") + for file in files_list: + print(file) + render_output(model_dir, files_list, None) + add_outputs(mint_config_file, files_list) + + +def get_command_line(resource): + line = './run ' + inputs = resource.has_input + try: + parameters = resource.has_parameter + except: + parameters = None + outputs = resource.has_output + if inputs: + l = build_input(inputs) + line += " {}".format(l) + if outputs: + l = build_output(outputs) + line += " {}".format(l) + if parameters is not None: + l = build_parameter(parameters) + line += " {}".format(l) + return line + + +def build_input(inputs): + """ + Download or search the file. Loop the inputs (metadata) of Model Configuration or Model Configuration Setup + """ + line = "" + for _input in inputs: + _file_path = _input.has_fixed_resource[0]["value"][0] + _format = _input.format[0] if hasattr(_input, "format") else None + file_name = _file_path + position = _input.position[0] + line += " -i{} {}".format(position, file_name) + return line diff --git a/src/mic/component/initialization.py b/src/mic/component/initialization.py new file mode 100644 index 0000000..06eb4ff --- /dev/null +++ b/src/mic/component/initialization.py @@ -0,0 +1,119 @@ +import shutil +from pathlib import Path +from typing import List + +import click +from jinja2 import Environment, PackageLoader, select_autoescape +from mic.component.python3 import freeze +from mic.constants import * + +env = Environment( + loader=PackageLoader('mic', 'templates'), + autoescape=select_autoescape(['html', 'xml']), + trim_blocks=False, + lstrip_blocks=False +) + + +def create_directory(parent_directory: Path, name: str): + parent_directory = parent_directory / name + if parent_directory.exists(): + shutil.rmtree(parent_directory) + src = parent_directory / SRC_DIR + docker = parent_directory / DOCKER_DIR + data = parent_directory / DATA_DIR + src.mkdir(parents=True) + docker.mkdir(parents=True) + data.mkdir(parents=True) + click.secho("Created: {}".format(src.absolute()), fg="green") + click.secho("Created: {}".format(docker.absolute()), fg="green") + click.secho("Created: {}".format(data.absolute()), fg="green") + return parent_directory + + +def render_gitignore(directory: Path): + template = env.get_template(GITIGNORE_FILE) + gitignore_file = directory / GITIGNORE_FILE + + with open(gitignore_file, "w") as gi: + ignore = render_template(template=template) + gi.write(ignore) + + gitignore_file.chmod(0o755) + click.secho("Created: {}".format(gitignore_file.absolute()), fg="green") + return gitignore_file + + +def render_run_sh(directory: Path, + inputs: dict, parameters: dict, + number_inputs: int = 0, number_parameters: int = 0) -> Path: + """ + + @param number_parameters: + @type number_parameters: + @param number_inputs: + @type number_inputs: + @param directory: + @type directory: + @param inputs: + @type inputs: + @param parameters: + @type parameters: + """ + template = env.get_template(RUN_FILE) + run_file = directory / SRC_DIR / RUN_FILE + with open(run_file, "w") as f: + content = render_template(template=template, inputs=inputs, parameters=parameters, + number_inputs=number_inputs, number_parameters=number_parameters, number_outputs=0) + f.write(content) + run_file.chmod(0o755) + return run_file + + +def render_io_sh(directory: Path, inputs: dict, parameters: dict, configs: list) -> Path: + template = env.get_template(IO_FILE) + data_dir = directory / DATA_DIR + run_file = directory / SRC_DIR / IO_FILE + with open(run_file, "w") as f: + content = render_template(template=template, inputs=inputs, + parameters=parameters, configs=[str(Path(directory / i).relative_to(data_dir)) for i in configs]) + f.write(content) + return run_file + + +def detect_framework(src_dir: Path) -> Framework: + return None + + +def render_dockerfile(model_directory: Path, language: Framework) -> Path: + template = env.get_template(DOCKER_FILE) + run_file = model_directory / DOCKER_DIR / DOCKER_FILE + with open(run_file, "w") as f: + content = render_template(template=template, language=language) + f.write(content) + # language_tasks(model_directory, language) + return run_file + + +def render_output(directory: Path, files: List[Path], compress: str) -> Path: + template = env.get_template(OUTPUT_FILE) + run_file = directory / SRC_DIR / OUTPUT_FILE + with open(run_file, "w") as f: + if files and compress: + content = render_template(template=template, files=files, compress=compress) + elif files: + content = render_template(template=template, files=files, compress=None) + else: + content = render_template(template=template, files=[], compress=None) + f.write(content) + return run_file + + +def language_tasks(directory, language): + if language == "python3": + run_file = directory / DOCKER_DIR / REQUIREMENTS_FILE + freeze(run_file) + + +def render_template(template, **kwargs): + return template.render(**kwargs) diff --git a/src/mic/component/python3.py b/src/mic/component/python3.py new file mode 100644 index 0000000..8e2fae1 --- /dev/null +++ b/src/mic/component/python3.py @@ -0,0 +1,10 @@ +import subprocess +import sys +from pathlib import Path + + +def freeze(requirements: Path) -> Path: + reqs = subprocess.check_output([sys.executable, '-m', 'pip', 'freeze']) + with open(requirements, "wb") as f: + f.write(reqs) + return requirements diff --git a/src/mic/config_yaml.py b/src/mic/config_yaml.py new file mode 100644 index 0000000..00d0a61 --- /dev/null +++ b/src/mic/config_yaml.py @@ -0,0 +1,313 @@ +import logging +import random +import re +import unicodedata +from pathlib import Path +from typing import List + +import yaml + +try: + from yaml import CLoader as Loader +except ImportError: + from yaml import Loader + +import click +from mic._makeyaml import make_yaml +from mic.constants import * + + +def slugify(value, allow_unicode=False): + """ + Convert to ASCII if 'allow_unicode' is False. Convert spaces to hyphens. + Remove characters that aren't alphanumerics, underscores, or hyphens. + Convert to lowercase. Also strip leading and trailing whitespace. + """ + value = str(value) + if allow_unicode: + value = unicodedata.normalize('NFKC', value) + else: + value = unicodedata.normalize('NFKD', value).encode('ascii', 'ignore').decode('ascii') + value = re.sub(r'[^\w\s-]', '', value.lower()).strip() + return re.sub(r'[-\s]+', '-', value) + + +def random_parameter(): + return 0 + + +def create_config_file_yaml(model_path: Path) -> Path: + config_yaml_path = model_path / CONFIG_YAML_NAME + if model_path.exists(): + click.secho("Searching files in the directory {}".format(model_path)) + else: + click.secho("Failed: Directory {} doesn't exist".format(model_path), fg="red") + exit(1) + spec = {} + write_step(config_yaml_path, spec, step=1) + + +def get_spec(config_yaml_path: Path) -> dict: + spec = yaml.load(config_yaml_path.open(), Loader=Loader) + return spec + + +def get_key_spec(config_yaml_path: Path, key: str): + spec = yaml.load(config_yaml_path.open(), Loader=Loader) + if key in spec: + return spec[key] + return None + + +def write_spec(config_yaml_path: Path, key: str, value: object): + spec = yaml.load(config_yaml_path.open(), Loader=Loader) + spec[key] = value + write_to_yaml(config_yaml_path, spec) + + +def get_key_spec(config_yaml_path: Path, key: str): + spec = yaml.load(config_yaml_path.open(), Loader=Loader) + if key in spec: + return spec[key] + return None + + +def write_spec(config_yaml_path: Path, key: str, value: object): + spec = yaml.load(config_yaml_path.open(), Loader=Loader) + spec[key] = value + write_to_yaml(config_yaml_path, spec) + + +def write_step(config_yaml_path: Path, spec: dict, step: int): + spec[STEP_KEY] = step + write_to_yaml(config_yaml_path, spec) + + +def write_docker_image(config_yaml_path: Path, spec: dict, image_name: str): + spec[DOCKER_KEY] = {NAME_KEY : image_name} + write_to_yaml(config_yaml_path, spec) + + +def write_docker_image(config_yaml_path: Path, spec: dict, image_name: str): + spec[DOCKER_KEY] = {NAME_KEY: image_name} + write_to_yaml(config_yaml_path, spec) + + +def fill_config_file_yaml(config_yaml_path: Path, data_dir: Path, parameters: int) -> Path: + directory = config_yaml_path.parent + if data_dir.exists(): + click.secho("Searching files in the directory {}".format(data_dir)) + else: + click.secho("Failed: Directory {} doesn't exist".format(data_dir), fg="red") + exit(1) + try: + spec = {} + input_files = [] + for x in data_dir.iterdir(): + if not x.name.startswith('.'): + input_files.append(x) + if input_files: + spec[INPUTS_KEY] = {} + + if parameters: + spec[PARAMETERS_KEY] = {} + + for index, item in enumerate(input_files): + name = slugify(str(item.name).replace('.', "_")) + spec[INPUTS_KEY][name] = {} + spec[INPUTS_KEY][name][PATH_KEY] = str(item.relative_to(directory)) + spec[INPUTS_KEY][name][DEFAULT_DESCRIPTION_KEY] = "" + + except Exception as e: + logging.error(e, exc_info=True) + click.secho("Failed: Error message {}".format(e), fg="red") + exit(1) + + try: + for parameter in range(0, parameters): + name = "parameter{}".format(parameter + 1) + spec[PARAMETERS_KEY][name] = {} + spec[PARAMETERS_KEY][name][DEFAULT_VALUE_KEY] = random_parameter() + spec[PARAMETERS_KEY][name][DEFAULT_DESCRIPTION_KEY] = "" + + write_step(config_yaml_path, spec, step=2) + add_comment(config_yaml_path, DEFAULT_VALUE_KEY, DEFAULT_PARAMETER_COMMENT) + add_comment(config_yaml_path, DEFAULT_DESCRIPTION_KEY, DEFAULT_DESCRIPTION_MESSAGE) + except Exception as e: + logging.error(e, exc_info=True) + click.secho("Failed: Error message {}".format(e), fg="red") + exit(1) + click.secho("MIC has added the parameters and inputs into the {}".format(MIC_CONFIG_FILE_NAME), fg="green") + click.secho("You can see the changes {}".format(config_yaml_path.absolute()), fg="green") + return config_yaml_path + + +def write_to_yaml(config_yaml_path: Path, spec): + """ + This function makes sure that the comments get saved when writing new data to the yaml file + @param config_yaml_path: path + @param spec: data for yaml + """ + comments = [] + if config_yaml_path.exists(): + comments = get_comment_list(config_yaml_path) + + with open(config_yaml_path, 'w') as f: + yaml.dump(spec, f, sort_keys=False) + + # yaml.dump will override comments in original yaml file. this will replace them + for i in comments: + new_line = False + if i['value'] is None or i['value'] == "": + new_line = True + + add_comment_by_line(config_yaml_path, i['line_number'], new_line, i['comment']) + + +def get_comment_list(config_yaml_path: Path): + """ + Return list of all the yaml values that have comments. This is needed becasue yaml.dump will erase any comments in + the yaml + @param config_yaml_path: path to yaml + @type config_yaml_path: Path + @return: list + """ + all_comments = [] + count = 0 + with open(config_yaml_path, "r") as file: + for line in file: + count += 1 + value_comment_line = {'value': "", 'line_number': count, 'comment': ""} + if "#" in line: + for i in line.split(" "): + if ":" in i: + value_comment_line['value'] = i + + curr_comment = line.split("#") + curr_comment.pop(0) + curr_comment = "#" + "#".join(curr_comment).replace("\n", "") + value_comment_line['comment'] = curr_comment + value_comment_line['line_number'] = count + if value_comment_line not in all_comments: + # deep copy + all_comments.append({'value': value_comment_line['value'], + 'line_number': value_comment_line['line_number'], + 'comment': value_comment_line['comment']}) + + return all_comments + + +def add_comment_by_line(config_yaml_path: Path, line_number, insert_new_line, comment): + """ + Adds comment to yaml file from line number + + @param config_yaml_path: path + @param line_number: line number comment is on + @param insert_new_line: If the comment was on a line without a value. Added comment needs to insert on new line + @param comment: comment to append + @return: + """ + new_file = [] + count = 0 + with open(config_yaml_path, "r") as file: + for line in file: + count += 1 + if line_number == count: + # make sure comment character is in comment + if "#" not in comment: + comment = "# " + comment + if not insert_new_line: + new_file.append(line.replace("\n", " " + comment + "\n")) + else: + new_file.append(comment + "\n") + new_file.append(line) + else: + new_file.append(line) + + if line_number > count: + new_file.append(comment + "\n") + + with open(config_yaml_path, "w") as file: + file.writelines(new_file) + + +def add_comment(config_yaml_path: Path, value, comment): + """ + yaml does not natively support comments, so this workaround has to be implemented. This function reads through + the yaml file and looks for the value given, then appends a comment to the end + @param config_yaml_path: + @type config_yaml_path: Path + @param value: name of field to append comment to + @type value: str + @param comment: comment to append + @type comment: str + """ + new_file = [] + with open(config_yaml_path, "r") as file: + for line in file: + # if the value is in line and line doesnt have comment + if value in line and "#" not in line: + # make sure comment character is in comment + if "#" not in comment: + comment = "# " + comment + + new_file.append(line.replace("\n", " " + comment + "\n")) + else: + new_file.append(line) + + with open(config_yaml_path, "w") as file: + file.writelines(new_file) + + +def add_outputs(config_yaml_path: Path, outputs: List[Path]): + spec = yaml.load(config_yaml_path.open(), Loader=Loader) + spec[OUTPUTS_KEY] = {} + for x in outputs: + name = slugify(str(x).replace('.', "_")) + spec[OUTPUTS_KEY][name] = {'path': str(x)} + spec[OUTPUTS_KEY][name][DEFAULT_DESCRIPTION_KEY] = "" + try: + write_to_yaml(config_yaml_path, spec) + add_comment(config_yaml_path, DEFAULT_DESCRIPTION_KEY, DEFAULT_DESCRIPTION_MESSAGE) + except Exception as e: + click.secho("Failed: Error message {}".format(e), fg="red") + for item in spec[OUTPUTS_KEY]: + click.secho("Added: {} as a output".format(item), fg="green") + + +def add_configuration_files(config_yaml_path: Path, configurations: tuple): + spec = yaml.load(config_yaml_path.open(), Loader=Loader) + spec[CONFIG_FILE_KEY] = [str(Path(x).relative_to(config_yaml_path.parent)) for x in list(configurations)] + + try: + write_to_yaml(config_yaml_path, spec) + except Exception as e: + click.secho("Failed: Error message {}".format(e), fg="red") + for item in spec[CONFIG_FILE_KEY]: + click.secho("Added: {} as a configuration file".format(item), fg="green") + + +def get_configuration_files(config_yaml_path: Path): + spec = yaml.load(config_yaml_path.open(), Loader=Loader) + return spec[CONFIG_FILE_KEY] + + +def get_inputs_parameters(config_yaml_path: Path) -> (dict, dict, dict): + spec = yaml.load(config_yaml_path.open(), Loader=Loader) + inputs = spec[INPUTS_KEY] if INPUTS_KEY in spec else None + parameters = spec[PARAMETERS_KEY] if PARAMETERS_KEY in spec else None + outputs = spec[OUTPUTS_KEY] if OUTPUTS_KEY in spec else None + configs = spec[CONFIG_FILE_KEY] if CONFIG_FILE_KEY in spec else [] + return inputs, parameters, outputs, configs + + +def get_numbers_inputs_parameters(config_yaml_path: Path) -> (int, int, int): + spec = yaml.load(config_yaml_path.open(), Loader=Loader) + number_inputs = len(spec[INPUTS_KEY].keys()) if INPUTS_KEY in spec else 0 + number_parameters = len(spec[PARAMETERS_KEY].keys()) if PARAMETERS_KEY in spec else 0 + number_outputs = len(spec[OUTPUTS_KEY].keys()) if OUTPUTS_KEY in spec else 0 + return number_inputs, number_parameters, number_outputs + + +def create_file_yaml_basic(config_yaml_path: Path): + make_yaml(config_yaml_path) diff --git a/src/mic/constants.py b/src/mic/constants.py new file mode 100644 index 0000000..994085d --- /dev/null +++ b/src/mic/constants.py @@ -0,0 +1,62 @@ +from enum import Enum + +CONFIG_FILE = "config.json" +CONFIG_YAML_NAME = "mic.yaml" +INPUTS_KEY = "inputs" +PARAMETERS_KEY = "parameters" +CONFIG_FILE_KEY = "configs" +STEP_KEY = "step" +OUTPUTS_KEY = "outputs" +NAME_KEY = "name" +DEFAULT_DESCRIPTION_KEY = "description" +PATH_KEY = "path" +DEFAULT_VALUE_KEY = "default_value" +DATA_DIRECTORY_NAME = "data" +RUN_FILE = "run" +IO_FILE = "io.sh" +OUTPUT_FILE = "output.sh" +DOCKER_FILE = "Dockerfile" +SRC_DIR = "src" +DOCKER_DIR = "docker" +MIC_CONFIG_FILE_NAME = "MIC configuration file" +DATA_DIR = "data" +REQUIREMENTS_FILE = "requirements.txt" +EXECUTIONS_DIR = "executions" +TOTAL_STEPS = 8 +MINT_COMPONENT_ZIP = "mint_component" +GIT_TOKEN_KEY = "git_token" +GIT_USERNAME_KEY = "git_username" +DOCKER_KEY = "docker_image" +LAST_EXECUTION_DIR = "last_execution_dir" +REPO_KEY = "github_repo_url" +VERSION_KEY = "version" +DOCKER_USERNAME_KEY = "dockerhub_username" +MINT_COMPONENT_KEY = "mint_component_url" +MINT_INSTANCE = "https://w3id.org/okn/i/mint/" + +TYPE_PARAMETER = "https://w3id.org/okn/o/sd#Parameter" +TYPE_MODEL_CONFIGURATION = "https://w3id.org/okn/o/sdm#ModelConfiguration" +TYPE_DATASET = "https://w3id.org/okn/o/sd#DatasetSpecification" +TYPE_SOFTWARE_IMAGE = "https://w3id.org/okn/o/sd#SoftwareImage" +TYPE_SOFTWARE_VERSION = "https://w3id.org/okn/o/sd#SoftwareVersion" +GITIGNORE_FILE = ".gitignore" +DEFAULT_PARAMETER_COMMENT = "# value added by MIC. Replace with your own default value" +DEFAULT_DESCRIPTION_MESSAGE = "# insert description left of this comment" + +class Framework(Enum): + PYTHON37 = ("python37", "mintproject/python37:20.5.1") + CONDA = ("conda", "mintproject/conda:20.5.1") + GENERIC = ("general", "mintproject/generic:20.5.1") + + def __init__(self, label, image): + self.label = label + self.image = image + + def __str__(self): + return self.label + + +def handle(value): + for i in Framework: + if value == i.label: + return i diff --git a/src/mic/credentials.py b/src/mic/credentials.py index 13ce24f..794a25c 100644 --- a/src/mic/credentials.py +++ b/src/mic/credentials.py @@ -23,7 +23,7 @@ def get_credentials(profile: str) -> dict: raise ValueError("Profile doesn't exists") -def configure_credentials(server, username, password, profile): +def configure_credentials(server, username, password, git_username, git_token, name, email, dockerhub_username, profile): credentials_file = pathlib.Path( os.getenv("MINT_CREDENTIALS_FILE", __DEFAULT_MINT_API_CREDENTIALS_FILE__) ).expanduser() @@ -38,10 +38,68 @@ def configure_credentials(server, username, password, profile): credentials[profile] = { "server": server, "username": username, - "password": password + "password": password, + "git_username": git_username, + "git_token": git_token, + "name": name, + "email": email, + "dockerhub_username": dockerhub_username, } with credentials_file.open("w") as fh: credentials_file.parent.chmod(0o700) credentials_file.chmod(0o600) credentials.write(fh) + + +def print_list_credentials(profile, short): + + credentials_file = pathlib.Path( + os.getenv("MINT_CREDENTIALS_FILE", __DEFAULT_MINT_API_CREDENTIALS_FILE__) + ).expanduser() + credentials = configparser.ConfigParser() + if credentials_file.exists(): + credentials.read(credentials_file) + else: + click.secho("WARNING: The profile doesn't exists. To configure it, run:\nmic configure -p {}".format(profile), + fg="yellow") + + profile_list = [] + + # list all profiles if none given + if profile is None: + for p in credentials: + # configparser has both DEFAULT and default read, no need to get both + if p is not "DEFAULT": + profile_list.append(get_credentials(p)) + # list given profile + else: + try: + profile_list.append(get_credentials(profile)) + except KeyError as e: + click.secho( + "WARNING: That profile doesn't exists. To configure it, run:\nmic configure -p {}".format(profile), + fg="yellow") + + click.echo("\n== Profiles ==") + if not short: + click.echo("") + for prof in profile_list: + # there is no way to get the key from the prof obj, so I have to manually format the tostring + click.echo("[{}]".format(prof.__str__().split(" ")[1].split(">")[0])) + + # Only show details if short is not used + if not short: + for field in prof: + # Dont print password or token + if field != "password" and field != "git_token": + click.echo(" {}: {}".format(field, prof[field])) + # Dont print full token for security reasons + elif field == "git_token": + # Its safe to print if its obviously not a github token + if len(prof[field]) < 6: + click.echo(" {}: {}".format(field, prof[field])) + else: + click.echo(" {}: Ending in \"...{}\"".format(field, (prof[field])[-5:])) + + click.echo("\n") diff --git a/src/mic/file.py b/src/mic/file.py index 3741d04..f8fc03a 100644 --- a/src/mic/file.py +++ b/src/mic/file.py @@ -1,5 +1,6 @@ import json import logging + import click @@ -20,22 +21,20 @@ def clean_null_terms(d): return d -def save(request): +def save(request, file_name=None): """ Function to save the current request as a JSON file :param request: JSON to save :return: """ try: - file_name = click.prompt('Enter the file name to save (without extension): ') - file_name += '.json' + if file_name is None: + file_name = click.prompt('Enter the file name to save (without extension): ') + file_name += '.json' # Remove nulls request_dump = clean_null_terms(request) with open(file_name, 'w') as outfile: - json.dump(request_dump, outfile) - print('File saved successfully') - # this will show status if saved. - # click.confirm('File saved successfully. Do you want to continue editing?', abort=True) + json.dump(request_dump, outfile, indent=4) except Exception as err: logging.info(err, exc_info=True) click.secho(f"An error occurred when saving the file", fg="red") diff --git a/src/mic/model_catalog_utils.py b/src/mic/model_catalog_utils.py index 48acd3c..214dc42 100644 --- a/src/mic/model_catalog_utils.py +++ b/src/mic/model_catalog_utils.py @@ -1,3 +1,5 @@ +import ast +import json import logging import click @@ -26,9 +28,9 @@ def get_label_from_response(response): else: labels.append(None) else: - if resource.label: + if hasattr(resource, "label") and resource.label: labels.append(resource.label[0]) - elif resource.id: + elif hasattr(resource, "id") and resource.id: labels.append(resource.id) else: labels.append(None) diff --git a/src/mic/publisher/__init__.py b/src/mic/publisher/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/mic/publisher/docker.py b/src/mic/publisher/docker.py new file mode 100644 index 0000000..0b63fb3 --- /dev/null +++ b/src/mic/publisher/docker.py @@ -0,0 +1,29 @@ +import click +import docker +from mic.config_yaml import get_key_spec, write_spec +from mic.constants import DOCKER_KEY, DOCKER_USERNAME_KEY, VERSION_KEY +from mic.credentials import get_credentials + + +def publish_docker(mic_config_path, profile): + version = get_key_spec(mic_config_path, VERSION_KEY) + image_name = get_key_spec(mic_config_path, DOCKER_KEY) + credentials = get_credentials(profile) + click.secho("Publishing the Docker Image") + + try: + client = docker.from_env() + if DOCKER_USERNAME_KEY not in credentials: + exit(0) + username = credentials[DOCKER_USERNAME_KEY] + image = client.images.get(image_name) + image_name_without_version = image_name.split("/")[-1].split(':')[0] + repository = "{}/{}".format(username, image_name_without_version) + image.tag(repository, version) + client.images.push(repository, version) + except Exception as e: + raise e + docker_image = "{}:{}".format(repository, version) + click.secho("Docker Image: {}".format(docker_image)) + write_spec(mic_config_path, DOCKER_KEY, docker_image) + diff --git a/src/mic/publisher/github.py b/src/mic/publisher/github.py new file mode 100644 index 0000000..e9ceff1 --- /dev/null +++ b/src/mic/publisher/github.py @@ -0,0 +1,221 @@ +import datetime +import logging +import re +import shutil +from pathlib import Path + +import click +import pygit2 as pygit2 +import semver +from mic.config_yaml import write_spec +from distutils.version import StrictVersion +from github import Github +from mic.constants import MINT_COMPONENT_ZIP, GIT_TOKEN_KEY, GIT_USERNAME_KEY, SRC_DIR, REPO_KEY, VERSION_KEY, \ + MINT_COMPONENT_KEY +from mic.credentials import get_credentials + +author = pygit2.Signature('MIC Bot', 'bot@mint.isi.edu') + + +def create_local_repo_and_commit(model_directory: Path): + """ + Publish the directory on git + If the directory is not a git directory, create it + If the git directory doesn't have a remote origin, create a github repository + @param directory: + @type directory: + """ + try: + repo = get_or_create_repo(model_directory) + git_commit(repo) + except Exception as e: + raise e + + +def push(model_directory: Path, mic_config_path: Path, profile): + click.secho("Creating the git repository") + repo = get_or_create_repo(model_directory) + click.secho("Compressing your code") + compress_src_dir(model_directory) + click.secho("Creating a new commit") + git_commit(repo) + click.secho("Creating or using the GitHub repository") + url = check_create_remote_repo(repo, profile, model_directory.name) + click.secho("Creating a new version") + _version = git_tag(repo, author) + click.secho("Pushing your changes to the server") + git_push(repo, profile, _version) + repo = get_github_repo(profile, model_directory.name) + for i in repo.get_contents(""): + if i.name == "{}.zip".format(MINT_COMPONENT_ZIP): + file = i + break + + write_spec(mic_config_path, REPO_KEY, url) + write_spec(mic_config_path, VERSION_KEY, _version) + write_spec(mic_config_path, MINT_COMPONENT_KEY, file.download_url) + click.secho("Repository: {}".format(url)) + click.secho("Version: {}".format(_version)) + + + + +def git_commit(repo): + repo.index.add_all() + repo.index.write() + tree = repo.index.write_tree() + parent = None + try: + parent = repo.revparse_single('HEAD') + except KeyError: + pass + + parents = [] + if parent: + parents.append(parent.oid.hex) + repo.create_commit('refs/heads/master', author, author, "automated mic", tree, parents) + + +def get_or_create_repo(model_path: Path): + return pygit2.Repository(pygit2.discover_repository(model_path)) if pygit2.discover_repository( + model_path) else pygit2.init_repository( + model_path, False) + + +def compress_src_dir(model_path: Path): + """ + Compress the directory src and create a zip file + """ + zip_file_name = model_path / MINT_COMPONENT_ZIP + tmp_dir = model_path / "{}_component".format(model_path.name) + shutil.copytree(model_path / SRC_DIR, tmp_dir / SRC_DIR) + zip_file_path = shutil.make_archive(zip_file_name.name, 'zip', root_dir=model_path, base_dir=tmp_dir.name) + return zip_file_path + +def check_create_remote_repo(repo, profile, model_name): + if "origin" in repo.remotes: + try: + return repo.remotes["origin"].url + except: + pass + try: + return repo.remotes["origin"].url + except: + repo = github_create_repo(profile, model_name) + url = repo.clone_url + repo.remotes.create("origin", url) + return url + + +def get_github_repo(profile, model_name): + git_token, git_username = github_config(profile) + g = Github(git_username, git_token) + github_login(g) + user = g.get_user() + return user.get_repo(model_name) + + +def git_add_remote(repo, url): + repo.remotes.create("origin", url) + + +def git_push(repo, profile, tag): + git_token, git_username = github_config(profile) + callbacks = pygit2.RemoteCallbacks(pygit2.UserPass(git_token, 'x-oauth-basic')) + remote = repo.remotes["origin"] + remote.push(['refs/heads/master'], callbacks=callbacks) + remote.push(['refs/tags/{}'.format(tag)], callbacks=callbacks) + + +def git_tag(repo, tagger): + """ + If there is a release, increment the version. + """ + version = get_next_tag(repo) + repo.create_tag(str(version), + repo.revparse_single('HEAD').oid.hex, + pygit2.GIT_OBJ_COMMIT, + tagger, + str(version)) + + click.secho("New version: {}".format(str(version))) + return str(version) + + +def get_next_tag(repo): + regex = re.compile('^refs/tags') + _tags = filter(lambda r: regex.match(r), repo.listall_references()) + tags = [tag.split('/')[-1] for tag in _tags] + tags.sort(key=StrictVersion) + today = datetime.date.today() + version_today = semver.VersionInfo.parse("{}.{}.{}".format(int(today.year) % 100, today.month, 1)) + if tags: + version_str = tags[-1] + try: + version = semver.VersionInfo.parse(version_str) + click.secho("Previous version {}".format(version)) + if int(version.minor) != today.month or int(version.major) != today.year % 100: + return version_today + else: + return version.bump_patch() + except ValueError as e: + logging.info(e) + pass + return version_today + + +def github_create_repo(profile, model_name): + """ + Publish the directory on git + If the directory is not a git directory, create it + If the git directory doesn't have a remote origin, create a github repository + @type profile: str + @param profile: the profile to use in the credentials file + @type: directory: Path + """ + + git_token, git_username = github_config(profile) + g = Github(git_username, git_token) + github_login(g) + user = g.get_user() + repo = None + try: + repo = user.get_repo(model_name) + except: + # TODO: github.GithubException.UnknownObjectException: 404 + # {"message": "Not Found", "documentation_url": "https://developer.github.com/v3/repos/#get"} + pass + if repo: + if not click.confirm("The repo {} exists. Do you want to use it?".format(model_name), default=True): + click.echo("Please rename the directory") + exit(0) + else: + repo = user.create_repo(model_name) + return repo + + +def github_config(profile): + # Try to get git username and token from credentials file + try: + credentials = get_credentials(profile) + git_username = credentials[GIT_USERNAME_KEY] + git_token = credentials[GIT_TOKEN_KEY] + except KeyError: + click.secho("WARNING: Could not find GitHub credentials in profile, " + "please run:\nmic configure -p {}".format(profile), fg="yellow") + exit(1) + return git_token, git_username + + +def github_login(g, debug=False): + try: + if g.get_user().login is None: + logging.error("User profile GitHub credentials are invalid. Please enter a valid token and username") + exit(1) + # I know its bad to except Exception but it doesnt catch it when I except TypeError, and the only way this *should* + # fail is if the credentials are bad so... + except Exception as e: + logging.error("User profile GitHub credentials are invalid. Please enter a valid token and username") + if debug: + click.secho(e, fg="yellow") + exit(1) diff --git a/src/mic/publisher/model_catalog.py b/src/mic/publisher/model_catalog.py new file mode 100644 index 0000000..a64a625 --- /dev/null +++ b/src/mic/publisher/model_catalog.py @@ -0,0 +1,214 @@ +import uuid +from pathlib import Path + +import click +import validators +from dame.cli_methods import create_sample_resource +from mic._menu import parse +from mic._utils import obtain_id +from mic.config_yaml import get_inputs_parameters, get_key_spec, DOCKER_KEY +from mic.constants import TYPE_PARAMETER, TYPE_DATASET, TYPE_SOFTWARE_IMAGE, MINT_COMPONENT_KEY, \ + TYPE_MODEL_CONFIGURATION, TYPE_SOFTWARE_VERSION, MINT_INSTANCE +from mic.drawer import print_choices +from mic.model_catalog_utils import get_label_from_response +from mic.resources.model import ModelCli +from mic.resources.model_configuration import ModelConfigurationCli +from mic.resources.software_version import SoftwareVersionCli +from modelcatalog import DatasetSpecification, ModelConfiguration, SoftwareImage, Parameter, Model, SoftwareVersion + + +def generate_uuid(): + return "https://w3id.org/okn/i/mint/{}".format(str(uuid.uuid4())) + + +def create_model_catalog_resource(mint_config_file, allow_local_path=True): + name = mint_config_file.parent.name + inputs, parameters, outputs, configs = get_inputs_parameters(mint_config_file) + + model_catalog_inputs = create_input_resource(allow_local_path, inputs, name) + model_catalog_outputs = create_output_resource(allow_local_path, outputs, name) + model_catalog_parameters = create_parameter_resource(parameters) + + image = get_key_spec(mint_config_file, DOCKER_KEY) + code = get_key_spec(mint_config_file, MINT_COMPONENT_KEY) + + model_configuration = ModelConfiguration(type=[TYPE_MODEL_CONFIGURATION], + label=[str(name)], + has_input=model_catalog_inputs, + has_output=model_catalog_outputs, + has_parameter=model_catalog_parameters, + ) + if allow_local_path: + return model_configuration + + if image is None: + click.secho("Failed to publish. Missing information DockerImage") + else: + software_image = SoftwareImage(label=[image], type=[TYPE_SOFTWARE_IMAGE]) + model_configuration.has_software_image = [software_image] + + if code is None: + click.secho("Failed to publish. Missing information zip file") + else: + model_configuration.has_component_location = [code] + return model_configuration + + +def create_parameter_resource(parameters): + model_catalog_parameters = [] + position = 1 + for key, item in parameters.items(): + _parameter = Parameter(id=generate_uuid(), label=[key], position=[position], type=[TYPE_PARAMETER]) + _parameter.has_default_value = [item["default_value"]] + model_catalog_parameters.append(_parameter) + position += 1 + if not model_catalog_parameters: + return None + return model_catalog_parameters + + +def create_output_resource(allow_local_path, outputs, name): + position = 1 + response = [] + if outputs: + for key, item in outputs.items(): + try: + _format = item["path"].split('.')[-1] + except: + _format = "unknown" + _input = DatasetSpecification(label=[key], has_format=[_format], position=[position], type=[TYPE_DATASET]) + if allow_local_path: + p = Path(name) / item["path"] + create_sample_resource(_input, str(p.resolve())) + response.append(_input) + position += 1 + response = response if response else None + return response + + +def create_input_resource(allow_local_path, inputs, name): + model_catalog_inputs = [] + position = 1 + for key, item in inputs.items(): + try: + if Path(item["path"]).is_dir(): + _format = "zip" + else: + _format = item["path"].name.split('.')[-1] + except: + _format = "unknown" + _input = DatasetSpecification(label=[key], has_format=[_format], position=[position], type=[TYPE_DATASET]) + if allow_local_path: + p = Path(name) / item["path"] + create_sample_resource(_input, str(p.resolve())) + model_catalog_inputs.append(_input) + position += 1 + + if not model_catalog_inputs: + return None + return model_catalog_inputs + + +def publish_model_configuration(model_configuration, profile): + model_configuration_cli = ModelConfigurationCli(profile=profile) + api_response_mc = model_configuration_cli.post(model_configuration) + + if not validators.url(api_response_mc.id): + api_response_mc.id = "{}{}".format(MINT_INSTANCE, api_response_mc.id) + print(api_response_mc.id) + click.echo("A model component must be associated with a model") + model_cli = ModelCli(profile=profile) + models = model_cli.get() + labels = get_show_models(models, "models") + if click.confirm("Do you want to use an existing model?", default=True): + api_response = handle_existing_model(profile, api_response_mc, labels, model_cli) + + else: + # todo: change to api_response_mc + api_response = create_new_model(model_cli, api_response_mc) + click.secho("Your Model Component has been published", fg="green") + return api_response, api_response_mc + + +def handle_existing_model(profile, api_response_mc, labels, model_cli): + models = model_cli.get() + choice = click.prompt("Please select the model to use", + default=1, + show_choices=False, + type=click.Choice(list(range(1, len(labels) + 1))), + value_proc=parse + ) + selected_model = models[choice - 1] + software_version_cli = SoftwareVersionCli(profile) + + labels = get_show_models_version(selected_model.has_version, software_version_cli) + software_version = handle_new_existing_software_version(labels, api_response_mc, selected_model, + software_version_cli) + + if selected_model.has_version: + selected_model.has_version.append(software_version) + else: + selected_model.has_version = [software_version] + print(selected_model) + return model_cli.put(selected_model) + + +def handle_new_existing_software_version(labels, api_response_mc, selected_model, software_version_cli): + if click.confirm("Do you want to use an existing version?", default=True): + choice = click.prompt("Select enter the number of version to use", + default=1, + show_choices=False, + type=click.Choice(list(range(1, len(labels) + 1))), + value_proc=parse + ) + model_version = selected_model.has_version[choice - 1] + model_version = software_version_cli.get_one(obtain_id(model_version.id)) + click.confirm(model_version.has_configuration) + + if model_version.has_configuration: + model_version.has_configuration = model_version.has_configuration.append(api_response_mc) + else: + model_version.has_configuration = [api_response_mc] + print(model_version.has_configuration) + return software_version_cli.put(model_version) + else: + click.echo("Please, enter the information about the new version") + _version = click.prompt("Version of the model") + description = click.prompt("A short description of the version") + software_version = SoftwareVersion(label=[_version], + description=[description], + type=[TYPE_SOFTWARE_VERSION], + has_version_id=[_version], + has_configuration=[api_response_mc]) + return software_version_cli.post(software_version) + + +def create_new_model(model_cli, model_configuration): + click.echo("Please enter the information about the Model") + name = click.prompt("Name of the model") + model_description = click.prompt("A short description of the Model") + _version = click.prompt("Version of the model") + version_description = click.prompt("A short description of the version") + new_model = Model(label=[name], + has_version=[SoftwareVersion(label=[_version], + description=[version_description], + type=[TYPE_SOFTWARE_VERSION], + has_version_id=[_version], + has_configuration=[model_configuration]) + ], + description=[model_description]) + return model_cli.post(new_model) + + +def get_show_models_version(resources, software_version_cli): + labels = get_label_from_response([software_version_cli.get_one(obtain_id(i.id)) for i in resources]) + click.secho("Existing versions are:") + print_choices(labels) + return labels + + +def get_show_models(resources, resource_name): + labels = get_label_from_response(resources) + click.secho("Existing {} are:".format(resource_name)) + print_choices(labels) + return labels diff --git a/src/mic/resources/model.py b/src/mic/resources/model.py index d8ef45d..bca4776 100644 --- a/src/mic/resources/model.py +++ b/src/mic/resources/model.py @@ -1,5 +1,6 @@ import logging import modelcatalog +from dame.utils import obtain_id from mic._mappings import mapping_person, mapping_model, mapping_software_version, mapping_image from modelcatalog import ApiException, Model import click @@ -51,10 +52,22 @@ def get(self): def post(self, request): api, username = get_api(profile=self.profile) api_instance = modelcatalog.ModelApi(api) - model = Model(**request) + model = Model(**request) if isinstance(request, dict) else request try: api_response = api_instance.models_post(username, model=model) return api_response except ApiException as e: logging.error("Exception when calling ModelConfigurationSetupApi->modelconfigurationsetups_post: %s\n" % e) - raise e \ No newline at end of file + raise e + + def put(self, request): + model_id = obtain_id(request.id) + api, username = get_api(profile=self.profile) + api_instance = modelcatalog.ModelApi(api) + model = Model(**request) if isinstance(request, dict) else request + + try: + # Update a Model + return api_instance.models_id_put(model_id, username, model=model) + except ApiException as e: + print("Exception when calling ModelApi->models_id_put: %s\n" % e) \ No newline at end of file diff --git a/src/mic/resources/model_configuration.py b/src/mic/resources/model_configuration.py index 276c36e..d0c057f 100644 --- a/src/mic/resources/model_configuration.py +++ b/src/mic/resources/model_configuration.py @@ -38,8 +38,7 @@ def get(self): def post(self, request): api, username = get_api(profile=self.profile) api_instance = modelcatalog.ModelConfigurationApi(api) - model_configuration = ModelConfiguration(**request) - + model_configuration = ModelConfiguration(**request) if isinstance(request, dict) else request try: api_response = api_instance.modelconfigurations_post(username, model_configuration=model_configuration) return api_response diff --git a/src/mic/resources/software_version.py b/src/mic/resources/software_version.py index 906e884..e22a6a3 100644 --- a/src/mic/resources/software_version.py +++ b/src/mic/resources/software_version.py @@ -5,7 +5,7 @@ from mic.model_catalog_utils import get_api from mic._mappings import mapping_model_configuration -from modelcatalog import ApiException +from modelcatalog import ApiException, SoftwareVersion from mic.resources.model_configuration import ModelConfigurationCli RESOURCE = "Software Version" @@ -23,6 +23,18 @@ class SoftwareVersionCli: def __init__(self, profile=None): self.profile = profile + def get_one(self, _id): + api, username = get_api(profile=self.profile) + api_instance = modelcatalog.SoftwareVersionApi(api) + try: + # List all Person entities + print(_id) + api_response = api_instance.softwareversions_id_get(id=_id, username=username) + return api_response + except ApiException as e: + raise e + + def get(self): api, username = get_api(profile=self.profile) api_instance = modelcatalog.SoftwareVersionApi(api) @@ -47,3 +59,14 @@ def put(self, software_version): "Exception when calling SoftwareVersionApi->softwareversions_id_put: %s\n" % e) raise e return api_response + + def post(self, request): + api, username = get_api(profile=self.profile) + api_instance = modelcatalog.SoftwareVersionApi(api) + software_version = SoftwareVersion(**request) if isinstance(request, dict) else request + try: + api_response = api_instance.softwareversions_post(username, software_version=software_version) + return api_response + except ApiException as e: + logging.error("Exception when calling ModelConfigurationSetupApi->modelconfigurationsetups_post: %s\n" % e) + raise \ No newline at end of file diff --git a/src/mic/templates/.gitignore b/src/mic/templates/.gitignore new file mode 100644 index 0000000..c35489b --- /dev/null +++ b/src/mic/templates/.gitignore @@ -0,0 +1,2 @@ +executions/ +data/ \ No newline at end of file diff --git a/src/mic/templates/Dockerfile b/src/mic/templates/Dockerfile new file mode 100644 index 0000000..943ab43 --- /dev/null +++ b/src/mic/templates/Dockerfile @@ -0,0 +1,16 @@ +FROM {{ language.image }} + +{% if "python" in language.label -%} +ADD requirements.txt /tmp/requirements.txt +pip install -r /tmp/requirements.txt +{% endif -%} + +{% if "conda" in language.label -%} +ADD environment.yml /tmp/environment.yml +RUN conda env update -f /tmp/environment.yml +{% endif -%} + +{% if "generic" in language.label -%} +#If you put an executable in the bin directory, uncomment the next file +#ADD bin/* /usr/bin/ +{% endif -%} \ No newline at end of file diff --git a/src/mic/templates/io.sh b/src/mic/templates/io.sh new file mode 100644 index 0000000..3f6c780 --- /dev/null +++ b/src/mic/templates/io.sh @@ -0,0 +1,109 @@ +#!/bin/bash + +# ----------------------------------------------- +# Option Parsing function for: +# -i<1..n> [files.. ] -p<1..n> {values} -o<1..n> [files.. ] +# {-iX fileX} {-pX valueX} {-oX fileX} +# +# +# - Please pass 3 Arguments to this script +# - Arg1: Number of Inputs expected +# - Arg2: Number of Parameters expected +# - Arg3: Number of Outputs expected +# ----------------------------------------------- + +INUM=$1; shift +PNUM=$1; shift +ONUM=$1; shift + +set_variables() +{ + for ((i=1; i<=INUM; i++)); do typeset ICOUNT$i=0; done + for ((i=1; i<=PNUM; i++)); do typeset PCOUNT$i=0; done + for ((i=1; i<=ONUM; i++)); do typeset OCOUNT$i=0; done +} + +IFLAG=(); +PFLAG=(); +OFLAG=(); +reset_flags() +{ + for ((j=1; j<=INUM; j++)); do IFLAG[$j]='0'; done + for ((j=1; j<=PNUM; j++)); do PFLAG[$j]='0'; done + for ((k=1; k<=ONUM; k++)); do OFLAG[$k]='0'; done +} + +set_variables +reset_flags + +while [ $# -gt 0 ] +do + case "$1" in + -i*) in=$(echo $1 | cut -di -f2); reset_flags; IFLAG[$in]='1';; + -p*) pa=$(echo $1 | cut -dp -f2); reset_flags; PFLAG[$pa]='1';; + -o*) op=$(echo $1 | cut -do -f2); reset_flags; OFLAG[$op]='1';; + --) shift; break;; + *) for((ind=1; ind<=INUM; ind++)); do + if [ "${IFLAG[$ind]}" = "1" ] + then + x="" + if [ "${INPUTS[$ind]}" != "" ]; then x="|"; fi + INPUTS[$ind]="${INPUTS[$ind]}$x$1" + fi + done + for((ind=1; ind<=PNUM; ind++)); do + if [ "${PFLAG[$ind]}" = "1" ] + then + x="" + if [ "${PARAM[$ind]}" != "" ]; then x="|"; fi + PARAMS[$ind]="${PARAMS[$ind]}$x$1" + fi + done + for((ind=1; ind<=ONUM; ind++)); do + if [ "${OFLAG[$ind]}" = "1" ] + then + x="" + if [ "${OUTPUTS[$ind]}" != "" ]; then x="|"; fi + OUTPUTS[$ind]="${OUTPUTS[$ind]}$x$1" + fi + done;; + esac + shift +done + +IFS='|' +for ((i=1; i<=INUM; i++)); do typeset INPUTS$i=$(echo ${INPUTS[$i]}); done +for ((i=1; i<=PNUM; i++)); do typeset PARAMS$i=$(echo ${PARAMS[$i]}); done +for ((i=1; i<=ONUM; i++)); do typeset OUTPUTS$i=$(echo ${OUTPUTS[$i]}); done +IFS=' ' + +{% if inputs -%} +{% for key, item in inputs.items() -%} +{{key}}=${INPUTS{{ loop.index }}} +export {{ key }} +{% endfor -%} +{% endif %} + + +## PARAMETERS VARIABLES +{% if parameters -%} +{% for key, item in parameters.items() -%} +{{key}}=${PARAMS{{loop.index }}} +export {{ key }} +{% endfor -%} +{% endif %} + + +## PARAMETERS VARIABLES +{% if configs -%} +{% for item in configs -%} + +find . -maxdepth 1 -name '*.zip' -execdir unzip '{}' ';' + + +cp {{item}} {{item}}.bk +envsubst < {{item}}.bk> {{item}} +{% endfor -%} +{% endif %} + + diff --git a/src/mic/templates/output.sh b/src/mic/templates/output.sh new file mode 100644 index 0000000..2149a0b --- /dev/null +++ b/src/mic/templates/output.sh @@ -0,0 +1,3 @@ +{% if compress -%} +zip {{ compress }} {{ files|join(' ') }} +{% endif %} diff --git a/src/mic/templates/run b/src/mic/templates/run new file mode 100644 index 0000000..8d9c8b0 --- /dev/null +++ b/src/mic/templates/run @@ -0,0 +1,29 @@ +#!/bin/bash +BASEDIR=$PWD +. $BASEDIR/io.sh {{ number_inputs }} {{ number_parameters }} {{number_outputs}} "$@" +CURDIR=`pwd` +set -e +## INPUTS VARIABLES +{% if inputs -%} +{% for key, item in inputs.items() -%} +{{key}}=${INPUTS{{ loop.index }}} +{% endfor -%} +{% endif %} + +## PARAMETERS VARIABLES +{% if parameters -%} +{% for key, item in parameters.items() -%} +{{key}}=${PARAMS{{loop.index }}} +{% endfor -%} +{% endif %} + +set -xe + +####### WRITE YOUR INVOCATION LINE AFTER THIS COMMENT + + + +####### WRITE YOUR INVOCATION LINE BEFORE THIS COMMENT +set -e +cd $BASEDIR +. $BASEDIR/output.sh diff --git a/src/mic/tests/test___main__.py b/src/mic/tests/test___main__.py new file mode 100644 index 0000000..5eca05b --- /dev/null +++ b/src/mic/tests/test___main__.py @@ -0,0 +1,181 @@ +import logging +import os + +from click.testing import CliRunner +from mic.__main__ import step1, step2, step3, step4, step5 +from mic.config_yaml import get_numbers_inputs_parameters +from mic.constants import * +from yaml import load + +try: + from yaml import CLoader as Loader, CDumper as Dumper +except ImportError: + from yaml import Loader, Dumper + +MODEL_NAME = "model" +PARAMETERS_2: int = 2 + + +def test_step1(tmp_path): + runner = CliRunner() + os.chdir(tmp_path) + try: + response = runner.invoke(step1, [MODEL_NAME]) + assert response.exit_code == 0 + except: + assert False + + +def test_step2(tmp_path): + runner = CliRunner() + os.chdir(tmp_path) + response = runner.invoke(step1, [MODEL_NAME]) + component_dir = tmp_path / MODEL_NAME + p = component_dir / DATA_DIRECTORY_NAME / "hello.txt" + p.write_text("test") + try: + response = runner.invoke(step2, ["-f", component_dir / CONFIG_YAML_NAME, "-p", 2]) + assert response.exit_code == 0 + except: + assert False + spec = load((component_dir / CONFIG_YAML_NAME).open(), Loader=Loader) + number_inputs, number_parameters, number_outputs = get_numbers_inputs_parameters(component_dir / CONFIG_YAML_NAME) + assert number_inputs == 1 + assert number_parameters == 2 + + + +def test_init_two_inputs(tmp_path): + runner = CliRunner() + os.chdir(tmp_path) + response = runner.invoke(step1, [MODEL_NAME]) + component_dir = tmp_path / MODEL_NAME + p = component_dir / DATA_DIRECTORY_NAME / "hello.txt" + p.write_text("test") + + p2 = component_dir / DATA_DIRECTORY_NAME / "hello2.txt" + p2.write_text("test") + + try: + response = runner.invoke(step2, ["-f", component_dir / CONFIG_YAML_NAME]) + assert response.exit_code == 0 + except: + assert False + spec = load((component_dir / CONFIG_YAML_NAME).open(), Loader=Loader) + number_inputs, number_parameters, number_outputs = get_numbers_inputs_parameters(component_dir / CONFIG_YAML_NAME) + assert number_inputs == 2 + assert number_parameters == 0 + + + +def test_init_two_inputs_zero_parameters(tmp_path): + runner = CliRunner() + os.chdir(tmp_path) + response = runner.invoke(step1, ["-n", MODEL_NAME]) + component_dir = tmp_path / MODEL_NAME + p = component_dir / DATA_DIRECTORY_NAME / "hello.txt" + p.write_text("test") + + p2 = component_dir / DATA_DIRECTORY_NAME / "hello2.txt" + p2.write_text("test") + + try: + response = runner.invoke(step2, ["-f", component_dir / CONFIG_YAML_NAME]) + assert response.exit_code == 0 + except: + assert False + number_inputs, number_parameters, number_outputs = get_numbers_inputs_parameters(component_dir / CONFIG_YAML_NAME) + assert number_inputs == 2 + assert number_parameters == 0 + + +def test_init_two_inputs_zero_parameters(tmp_path): + runner = CliRunner() + os.chdir(tmp_path) + response = runner.invoke(step1, [MODEL_NAME]) + component_dir = tmp_path / MODEL_NAME + p = component_dir / DATA_DIRECTORY_NAME / "hello.txt" + p.write_text("test") + + p2 = component_dir / DATA_DIRECTORY_NAME / "hello_dir" + p2.mkdir() + p3 = p2 / "hello.txt" + p3.write_text("test") + + try: + response = runner.invoke(step2, ["-f", component_dir / CONFIG_YAML_NAME]) + assert response.exit_code == 0 + except: + assert False + + number_inputs, number_parameters, number_outputs = get_numbers_inputs_parameters(component_dir / CONFIG_YAML_NAME) + assert number_inputs == 2 + assert number_parameters == 0 + + +def test_step3(tmp_path): + runner = CliRunner() + os.chdir(tmp_path) + response = runner.invoke(step1, [MODEL_NAME]) + component_dir = tmp_path / MODEL_NAME + p = component_dir / DATA_DIRECTORY_NAME / "hello.txt" + p.write_text("test") + + p2 = component_dir / DATA_DIRECTORY_NAME / "hello2.txt" + p2.write_text("test") + + try: + response = runner.invoke(step2, ["-f", component_dir / CONFIG_YAML_NAME]) + print(response.output) + assert response.exit_code == 0 + except: + assert False + try: + response = runner.invoke(step3, ["-f", component_dir / CONFIG_YAML_NAME]) + assert response.exit_code == 0 + except Exception as e: + print(e) + logging.error(e, exc_info=True) + assert False + + try: + response = runner.invoke(step4, ["-f", component_dir / CONFIG_YAML_NAME, str(p2)]) + assert response.exit_code == 0 + except: + assert False + + +def test_step5(tmp_path): + runner = CliRunner() + os.chdir(tmp_path) + response = runner.invoke(step1, [MODEL_NAME]) + component_dir = tmp_path / MODEL_NAME + input_config_path = component_dir / DATA_DIRECTORY_NAME / "this_is_config_file.txt" + input_config_path.write_text("0 0 {{parameter_1}} {{parameter_2}") + + input_data_path = component_dir / DATA_DIRECTORY_NAME / "this_is_data.txt" + input_data_path.write_text("test") + + try: + response = runner.invoke(step2, ["-f", component_dir / CONFIG_YAML_NAME, "-p", 2]) + assert response.exit_code == 0 + except: + assert False + try: + response = runner.invoke(step3, ["-f", component_dir / CONFIG_YAML_NAME]) + assert response.exit_code == 0 + except: + assert False + + try: + response = runner.invoke(step4, ["-f", component_dir / CONFIG_YAML_NAME, str(input_config_path)]) + assert response.exit_code == 0 + except: + assert False + + try: + response = runner.invoke(step5, ["-f", component_dir / CONFIG_YAML_NAME]) + assert response.exit_code == 0 + except Exception as e: + logging.error(e, exc_info=True) + assert False diff --git a/src/mic/tests/test__schema.py b/src/mic/tests/test__schema.py new file mode 100644 index 0000000..51e2074 --- /dev/null +++ b/src/mic/tests/test__schema.py @@ -0,0 +1,9 @@ +from mic._schema import get_schema, get_schema_version, schemaVersion + + +def test_get_schema(): + assert get_schema() + + +def test_get_schema_version(): + assert get_schema_version() == schemaVersion diff --git a/src/mic/tests/test_directory.py b/src/mic/tests/test_directory.py new file mode 100644 index 0000000..385133b --- /dev/null +++ b/src/mic/tests/test_directory.py @@ -0,0 +1,4 @@ +import pytest +from mic.component.initialization import render_run_sh + + diff --git a/src/mic/tests/test_executor.py b/src/mic/tests/test_executor.py new file mode 100644 index 0000000..f2fd85c --- /dev/null +++ b/src/mic/tests/test_executor.py @@ -0,0 +1,29 @@ +from datetime import datetime + +import pytest +from mic.component.executor import detect_news_file +from mic.constants import * + + +def test_detect_news_file(tmp_path): + p0 = tmp_path / "hello0.txt" + now = datetime.now().timestamp() + d1 = tmp_path / "subdir" + d1.mkdir() + d2 = tmp_path / SRC_DIR + d2.mkdir() + p1 = tmp_path / "hello1.txt" + p2 = tmp_path / "hello2.txt" + p3 = tmp_path / "hello3.txt" + p4 = tmp_path / "hello4.txt" + p5 = d1 / "hello5.txt" + p6 = d2 / "output.sh" + p7 = tmp_path / CONFIG_YAML_NAME + p1.write_text(" ") + p2.write_text(" ") + p3.write_text(" ") + p4.write_text(" ") + p5.write_text(" ") + p6.write_text(" ") + p7.write_text(OUTPUTS_KEY + ":") + detected = detect_news_file(tmp_path, tmp_path / CONFIG_YAML_NAME, now) diff --git a/src/mic/tests/test_initialization.py b/src/mic/tests/test_initialization.py new file mode 100644 index 0000000..8f231a6 --- /dev/null +++ b/src/mic/tests/test_initialization.py @@ -0,0 +1,6 @@ +import pytest +from mic.component.initialization import render_run_sh, create_directory + + +def test_create_directory(tmp_path): + assert create_directory(tmp_path, "test").exists() diff --git a/src/mic/tests/test_model_catalog_utils.py b/src/mic/tests/test_model_catalog_utils.py new file mode 100644 index 0000000..5c7d19c --- /dev/null +++ b/src/mic/tests/test_model_catalog_utils.py @@ -0,0 +1,6 @@ +from mic.model_catalog_utils import get_api + + +def test_get_api(): + get_api() + assert False diff --git a/tox.ini b/tox.ini index 80bd7a2..70eb564 100644 --- a/tox.ini +++ b/tox.ini @@ -1,10 +1,11 @@ [tox] -envlist = py37 +envlist = py37,py35 + [testenv] -commands = pytest --cov {envsitepackagesdir}/mic {posargs} +commands = pytest --cov src/mic {posargs} setenv = PYTHONPATH = {toxinidir}/src PYTHONUNBUFFERED = yes