From ed79ed925139885da9b1bea6cae57d529eddbb91 Mon Sep 17 00:00:00 2001 From: Maximiliano Osorio Date: Tue, 1 Jun 2021 10:56:25 -0400 Subject: [PATCH] Upload files to s3 (#347) * fix: removing github * fix: add s3 * fix: removing git references * fix: remove none general * fix: use label framework * fix: upload * fix: upload url * fix: unable to upload * fix: mic component zip * fix: if fails ends * fix: wrong path * fix: wrong key * fix: first version install.sh * fix: install * fix: install script * fix: improving start * fix: r must lower * fix: remove autodetect parameters * fix: file with windows encoding * fix: add install script * Revert "fix: remove autodetect parameters" This reverts commit a3c784ac334657bc785dae27f1e8ca4ead1424cc. * fix: message * fix: old mic with wrong reference * fix: remove references github from test * fix: remove github references docs * fix: remove deprecated docs * fix: remove some tests --- .vscode/settings.json | 2 +- docs/commands/configure.md | 32 +- .../02-Preparing your executable.md | 3 +- docs/model_configuration/03b-step1.md | 8 - .../deprecated/03-create-skeleton.md | 12 - .../deprecated/03-step1.md | 67 ---- .../deprecated/04-copy-your-data.md | 77 ---- .../deprecated/05-write-invocation.md | 61 --- .../deprecated/06-pass-parameters.md | 92 ----- .../deprecated/07-validate.md | 26 -- .../deprecated/08-outputs.md | 43 -- .../deprecated/09-publish.md | 10 - docs/model_configuration/deprecated/docker.md | 1 - docs/notebooks/upload_component.md | 2 +- docs/overview.md | 8 +- mkdocs.yml | 2 +- pytest.ini | 4 +- setup.py | 5 +- src/mic/__main__.py | 14 +- src/mic/cli_docs.py | 2 +- src/mic/click_encapsulate/commands.py | 32 +- src/mic/component/detect.py | 10 +- src/mic/component/initialization.py | 27 +- src/mic/config_yaml.py | 2 +- src/mic/constants.py | 7 +- src/mic/credentials.py | 16 +- src/mic/publisher/github.py | 365 ----------------- src/mic/publisher/model_catalog.py | 1 + src/mic/publisher/s3.py | 61 +++ src/mic/templates/Dockerfile | 1 + src/mic/templates/entrypoint.sh | 2 + src/mic/templates/install.sh | 375 ++++++++++++++++++ src/mic/tests/resources/254/mic/mic3.yaml | 5 +- src/mic/tests/resources/swat/mic/mic.yaml | 5 +- .../tests/resources/topoflow_dt/mic/mic.yaml | 5 +- src/mic/tests/test_model_catalog_utils.py | 32 +- 36 files changed, 517 insertions(+), 900 deletions(-) delete mode 100644 docs/model_configuration/deprecated/03-create-skeleton.md delete mode 100644 docs/model_configuration/deprecated/03-step1.md delete mode 100644 docs/model_configuration/deprecated/04-copy-your-data.md delete mode 100644 docs/model_configuration/deprecated/05-write-invocation.md delete mode 100644 docs/model_configuration/deprecated/06-pass-parameters.md delete mode 100644 docs/model_configuration/deprecated/07-validate.md delete mode 100644 docs/model_configuration/deprecated/08-outputs.md delete mode 100644 docs/model_configuration/deprecated/09-publish.md delete mode 100644 docs/model_configuration/deprecated/docker.md delete mode 100644 src/mic/publisher/github.py create mode 100644 src/mic/publisher/s3.py create mode 100644 src/mic/templates/install.sh diff --git a/.vscode/settings.json b/.vscode/settings.json index 66e0cb1..174c99a 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -4,7 +4,7 @@ "python.linting.pylintArgs": [ "--enable=W0614" ], - "python.pythonPath": "/home/mosorio/.pyenv/versions/mic/bin/python", + "python.pythonPath": "/home/mosorio/.pyenv/versions/mic_delete_git/bin/python", "python.testing.unittestEnabled": false, "python.testing.nosetestsEnabled": false, "python.testing.pytestEnabled": true diff --git a/docs/commands/configure.md b/docs/commands/configure.md index c0c78a0..a70d503 100644 --- a/docs/commands/configure.md +++ b/docs/commands/configure.md @@ -7,7 +7,7 @@ mic credentials [-p | --profile] [--server] [--username] [--password] [--name] ## Description -MIC uses several APIs (GitHub, DockerHub and MINT Model catalog) to upload model components. The MINT Model Catalog requires credentials for adding and modifying contents in the catalog. You can use the `credentials` command to configure a username and password for the [Model Catalog API](https://model-catalog-python-api-client.readthedocs.io/en/latest/endpoints/), GitHub and DockerHub. For ease of use, this command can also be used with no parameters, it will prompt the user to enter any required field not given. +MIC uses several APIs (DockerHub and MINT Model catalog) to upload model components. The MINT Model Catalog requires credentials for adding and modifying contents in the catalog. You can use the `credentials` command to configure a username and password for the [Model Catalog API](https://model-catalog-python-api-client.readthedocs.io/en/latest/endpoints/) and DockerHub. For ease of use, this command can also be used with no parameters, it will prompt the user to enter any required field not given. ## Options @@ -31,39 +31,11 @@ Password for Model Catalog - [required] Full name of the author - [required] -`--git_username ` - -Author's Github username - [required] - -`--git_token ` - -Authors's GitHub API Token. More information can be found in the [setting up GitHub credentials](#GitHubCreds) section below - [required] - `--dockerhub_username ` Username for dockerhub -## Setting up GitHub credentials - -!!! info - If you are transforming IPython Notebook, you can skip this section - -The `GitHub Username` field is the users GitHub username. If unknown the username can be found at [GitHub.com](https://github.com/). Once logged, in at the top right dropdown menu there will be a "signed in as **[username]**" - -The `GitHub Token` is the user's [personal access token](https://help.github.com/en/github/authenticating-to-github/creating-a-personal-access-token-for-the-command-line). To create a personal access token click [here](https://github.com/settings/tokens/new), or go to GitHub.com -> Settings -> Developer settings -> personal access token. Click **Generate new token** this will open the new personal access token page. The following options must be checked: - - - [x] repo: | Full control of private repositories - - [x] write:packages | Upload packages to github package registry - - [x] read:packages | Download packages from github package registry - - Writing "mic access token" under notes is also recommended - - When done click **Generate token** at the bottom of the page. Once the token is generated be sure to copy and save it in a secure location. Enter this key in the `GitHub API token` field when prompted. - -!!! warning - If this token is lost there is no way to recover it without generating a new one. - ## Setting up DockerHub credentials MIC will prompt you to add your user in [DockerHub](hub.docker.com/), a repository used for publishing Docker images. MIC will help you publish the computational dependencies of your model as a virtual image, giving you full control over the result. @@ -77,8 +49,6 @@ Username [mint@isi.edu]: Password: Name: Email: -GitHub Username: -GitHub API token: Docker Username: ``` diff --git a/docs/model_configuration/02-Preparing your executable.md b/docs/model_configuration/02-Preparing your executable.md index ff6d346..89d381c 100644 --- a/docs/model_configuration/02-Preparing your executable.md +++ b/docs/model_configuration/02-Preparing your executable.md @@ -28,7 +28,8 @@ If you would like to proceed with this option, the next step is to [prepare your If the software is not written in Python or cannot be practically encapsulated in a notebook, using the command line version of MIC might be easier. To do so: -1. Place your software code in a local directory so it can be invoked from a command line along with any input data and configuration files if applicable. MIC will test that the code can be executed and will do so three times during the encapsulation process. Therefore, we recommend that your execution can be completed in a manner of minutes, for example, by reducing the size of the dataset (e.g., only considering a few months instead of a year). Because of GitHub limitations, files should not exceed **100 MB** each. +1. Place your software code in a local directory so it can be invoked from a command line along with any input data and configuration files if applicable. MIC will test that the code can be executed and will do so three times during the encapsulation process. Therefore, we recommend that your execution can be completed in a manner of minutes, for example, by reducing the size of the dataset (e.g., only considering a few months instead of a year). + 2. The code should not contain any hardcoded paths or values for the input files/variables that you wish to expose. We recommend making them explicit in a configuration file or as parameters from the command line execution. ### Parameters diff --git a/docs/model_configuration/03b-step1.md b/docs/model_configuration/03b-step1.md index 2781dc9..2aad2e5 100644 --- a/docs/model_configuration/03b-step1.md +++ b/docs/model_configuration/03b-step1.md @@ -45,19 +45,11 @@ Successfully built 4950fcaa2d0d Successfully tagged test_192:latest You are in a Linux environment Debian distribution -We detect the following dependencies. - -- If you install new dependencies using `apt` or `apt-get`, remember to add them in Dockerfile mic\docker\Dockerfile -- If you install new dependencies using python. Before the step `upload` run: - -pip freeze > mic/docker/requirements.txt Please, run your Model Component. ``` As can be seen in the message above, MIC is creating an execution environment to make sure we capture the minimum set of dependencies needed for execution. Since we had a java executable, MIC already selected a Java environment. If we had python files, MIC would have promted us to select which version of Python to start from. -!!! warning - This command must **NOT** be executed on a folder already tracked by GitHub. ### Creating your own image diff --git a/docs/model_configuration/deprecated/03-create-skeleton.md b/docs/model_configuration/deprecated/03-create-skeleton.md deleted file mode 100644 index 38baaa3..0000000 --- a/docs/model_configuration/deprecated/03-create-skeleton.md +++ /dev/null @@ -1,12 +0,0 @@ -### Creating the directory skeleton for your first configuration - -You must create the directories and sub directories - -```bash -$ mic pkg step1 -``` - -MIC has created a directory `model_name`. -In this directory, there two subdirectories: -- data: Contains your data/inputs. -- src: Contains the invocation script. diff --git a/docs/model_configuration/deprecated/03-step1.md b/docs/model_configuration/deprecated/03-step1.md deleted file mode 100644 index cad31f9..0000000 --- a/docs/model_configuration/deprecated/03-step1.md +++ /dev/null @@ -1,67 +0,0 @@ -## Step 1: Set up a MIC directory structure and MIC file template - -This step is going to create: -The directory of your model component -Three subdirectories (data/, src/ and docker/) in the model component directory. -A MIC file template (mic.yaml). This file (which will be completed in subsequent steps) contains the executable information and metadata about your model component. - -### How to perform this step? - -To run this step, you must type the following command (where is the name of your model component): - -```bash -$ mic pkg step1 -``` - -For example, - -```bash -$ mic pkg step1 swat_precipitation_rates -Created: /Users/mosorio/tmp/swat_precipitation_rates/src -Created: /Users/mosorio/tmp/swat_precipitation_rates/docker -Created: /Users/mosorio/tmp/swat_precipitation_rates/data -Created: /Users/mosorio/tmp/swat_precipitation_rates/.gitignore -Searching files in the directory /Users/mosorio/tmp/swat_precipitation_rates -MIC has created the directories -You must add your data (files or directories) into the directory: /Users/mosorio/tmp/swat_precipitation_rates/data -``` -!!!warning - This command must **NOT** be executed on a folder already tracked by GitHub. - - -#### Expected Results - -After executing the previous command, MIC creates three directories and a MIC file (mic.yaml): - - -data/: It contains your data (now it will be empty). In step 2, you will have to copy your data in this directory. -src/: It contains your code and MIC Wrapper (i.e., the file that executes your code). -In step 3, MIC is going to help you generate the MINT Wrapper -In step 5, you are going to specify how to run your model in the MINT Wrapper (i.e., the series of commands). -docker/: It contains the required files to create the Docker Image (if everything goes well, you will not have to modify this directory). In `step 7`, MIC will populate this directory with the files that are needed to capture your computational infrastructure. - -The MIC file will have a single line at the moment:. - -```yaml -step: 1 -``` - - -#### Help command - -```bash -Usage: mic pkg step1 [OPTIONS] MODEL_COMPONENT_NAME - -This step is going to create: -The required directories -And the MIC file. The MIC file contains the metadata about your model component. - - mic pkg step1 - - The argument: `model_component_name` is the name of your model - component - -Options: - --help Show this message and exit. - -``` diff --git a/docs/model_configuration/deprecated/04-copy-your-data.md b/docs/model_configuration/deprecated/04-copy-your-data.md deleted file mode 100644 index 33b0a04..0000000 --- a/docs/model_configuration/deprecated/04-copy-your-data.md +++ /dev/null @@ -1,77 +0,0 @@ -## Identify your inputs and copy them - -You must copy your inputs into the directory `data`. - -!!! warning - Your code is not an input. - -An `input` can be: - -- A file in the directory `data` is one input. -- A directory in the directory `data` is one input (MIC is going create a zip file). - -Let's suppose that you have copied the following directory and file -- GLDAS_NOAH025_M.2.1/ - This is a directory -- prepicipitation_rates.txt - This is a file - - -## Identify your parameters - -Analysts may want to explore indicators values under different initial conditions. These are expressed as adjustable parameters of models. - - -Let's suppose that you have identified two parameters: -- start_year: -- end_year - - -## Creating `config.yaml` file - -Then, you must run the command: - -```bash -$ mic pkg step2 --inputs_dir data/ --number-parameters 2 -or -$ mic pkg step2 --number-parameters 2 -``` - -This command generates `config.yaml` file. This YAML file with the information about your model configuration - -```yaml -inputs: - gldas_noaho25_m.2.1: - path: data/GLDAS_NOAH025_M.2.1/ - prepicipitation_rates: - path: data/prepicipitation_rates.txt -parameters: - parameter1: - default_value: - parameter2: - default_value: -``` - -You **must** add: - - - A *default_value* for each parameter - -You **can** edit - - - The name of the parameters and inputs (Spaces are not admitted) - -### Creating the invocation code - -Then, we must generate the MINT wrapper to run your model - -You must pass the `MIC_CONFIG_FILE` (`config.yaml`) using the option (`-f`). - - -```bash -$ mic pkg step3 -f config.yaml -The invocation has been created. -``` - -!!! warning - If you edit the inputs or the parameters section in the `config.yaml` file, you must re-run ` mic pkg step3 -f config.yaml` - - -In the next step, you are going to learn how to run your models using the MINT Wrapper diff --git a/docs/model_configuration/deprecated/05-write-invocation.md b/docs/model_configuration/deprecated/05-write-invocation.md deleted file mode 100644 index 07bb920..0000000 --- a/docs/model_configuration/deprecated/05-write-invocation.md +++ /dev/null @@ -1,61 +0,0 @@ -## MINT wrapper - -You must add the invocation line of your model, - -### Types - -An invocation can be one line. - -For example, FloodSeverityIndex model. [Example](https://github.com/mintproject/MINT-WorkflowDomain/blob/master/WINGSWorkflowComponents/fsi-1.0.0/src/run#L19) - -```bash -python FloodSeverityIndex.py ./ GloFAS_FloodThreshold.nc [23,48,3,15] [2016,2017] True -``` - -Or multiple lines as the HAND model. [Example](https://github.com/mintproject/HAND-TauDEM/blob/master/hand_v2_mint_component/src/run#L27) - - -```bash -... -pitremove -z $1 -fel demfel.tif -dinfflowdir -fel demfel.tif -ang demang.tif -slp demslp.tif -d8flowdir -fel demfel.tif -p demp.tif -sd8 demsd8.tif -aread8 -p demp.tif -ad8 demad8.tif -nc -areadinf -ang demang.tif -sca demsca.tif -nc - -## Skeleton -slopearea -slp demslp.tif -sca demsca.tif -sa demsa.tif -d8flowpathextremeup -p demp.tif -sa demsa.tif -ssa demssa.tif -nc -python3 hand-thresh.py --resolution demfel.tif --output demthresh.txt -threshold -ssa demssa.tif -src demsrc.tif -thresh 500 - -streamnet -fel demfel.tif -p demp.tif -ad8 demad8.tif -src demsrc.tif -ord demord.tif -tree demtree.dat -coord demcoord.dat -net demnet.shp -w demw.tif -sw - -connectdown -p demp.tif -ad8 demad8.tif -w demw.tif -o outlets.shp -od movedoutlets.shp - -python3 hand-heads.py --network demnet.shp --output dangles.shp -python3 hand-weights.py --shapefile dangles.shp --template demfel.tif --output demwg.tif -``` - -What is the best option? That's is your decision. We provide flexibility. - - - - -### Adding your invocation line - -!!! info - The language of the run file is bash. - - -1. Open the file `src/run` -2. Add the invocation line(s) after the comment `# WRITE THE COMMAND LINE INVOCATION HERE.` - - -``` -# WRITE THE COMMAND LINE INVOCATION HERE -python FloodSeverityIndex.py ./ GloFAS_FloodThreshold.nc [23,48,3,15] [2016,2017] True -``` - - -On the next page, we are going to learn how to pass the parameters to your model. \ No newline at end of file diff --git a/docs/model_configuration/deprecated/06-pass-parameters.md b/docs/model_configuration/deprecated/06-pass-parameters.md deleted file mode 100644 index efe2dd8..0000000 --- a/docs/model_configuration/deprecated/06-pass-parameters.md +++ /dev/null @@ -1,92 +0,0 @@ -# Passing the parameter to your model - -Analysts may want to explore indicators values under different initial conditions. These are expressed as adjustable parameters and input variables of models. - -We identified two ways to pass the parameters to your model. To explain this, we are going to use example `config.yaml` - - -In this case, we have two basic parameters: - -- start_date -- end_date - -```yaml -... -parameters: - start_date: - default_value: 2010 - end_date: - default_value: 2012 -``` - - -## Arguments - -Some models read the parameters from the command line. For example, the invocation line for cycles is: - -```bash -python3 cycles-wrapper.py --start-year 2010 --end-year 2012 -``` - -Then, you must replace the value by the variable in the invocation line -```bash -python3 cycles-wrapper.py --start-year ${start_date} --end-year ${end_date} -``` - -## Configuration file - -Some models read the parameters from a configuration file. - -For example, the `SWAT+` model has the following invocation line. - -```bash -swatplus -``` - -SWAT uses configuration files to pass the parameters. To manipulate the simulation dates, SWAT has a file named `time.sim` - -``` - DAY_START YRC_START DAY_END YRC_END STEP - 0 2001 0 2003 0 -``` - -Then, we must open the file and replace the values with variables -```bash - DAY_START YRC_START DAY_END YRC_END STEP - 0 ${start_date} 0 ${end_date} 0 -``` - -And add the file as configuration file of the model. - -```bash -mic pkg step4 -f config.yaml [configuration_files]... -``` - -In the example, we must run -``` -mic pkg step4 -f config.yaml`src/time.sim` -``` - -And the `config.yaml` has been updated - -``` -```yaml -inputs: - gldas_noaho25_m.2.1 - path: data/GLDAS_NOAH025_M.2.1/ - pre - path: data/prepicipitation_rates.txt -parameters: - - name: start_date - default_value: 2010 - - name: end_date - default_value: 2012 -config_files - - src/time.sim -``` - -!!! warning - If you edit the inputs or the parameters section in the `config.yaml` file, you must re-run `mic model_configuration init config.yaml` - -!!! info - We are using a standard template language JINJA \ No newline at end of file diff --git a/docs/model_configuration/deprecated/07-validate.md b/docs/model_configuration/deprecated/07-validate.md deleted file mode 100644 index 1a40cfe..0000000 --- a/docs/model_configuration/deprecated/07-validate.md +++ /dev/null @@ -1,26 +0,0 @@ -It's time to run and validate your component. - -If you don't want to test the Docker Image, add the option `--no-docker` - -```bash -$ mic model_configuration validate hello_world --no-docker -``` - -The validation process is going to validate: -- Pass the parameters to your model -- The execution on your machine -- The creation of the Docker Image -- The execution on your machine using Docker - -```bash -$ mic model_configuration validate hello_world/ -[OK] Parameters -[OK] Execution without Docker -[OK] Extraction of dependencies -[OK] Build Docker Image -[OK] Execution using Docker - -Created: hello_world/hello_world.zip -``` - -The result is going to be a zip file. \ No newline at end of file diff --git a/docs/model_configuration/deprecated/08-outputs.md b/docs/model_configuration/deprecated/08-outputs.md deleted file mode 100644 index 14dd88e..0000000 --- a/docs/model_configuration/deprecated/08-outputs.md +++ /dev/null @@ -1,43 +0,0 @@ -## Identify the output files - -Now, we must identify the outputs of your model. - -!!! question - Why is it important? - You can add metadata about the outputs such as Units, formats, etc. - -The output of your models are in the directory src/ - -Let's suppose that the output files are the file `file.txt` and the directory `src/images` - -You must run: - -```bash -$ mic modelconfiguration outputs src/file.txt src/images -``` - -Then, the config file `config.yaml` has been updated - -```yaml -inputs: - - name: gldas_noaho25_m.2.1 - path: data/GLDAS_NOAH025_M.2.1/ - - name: - path: data/prepicipitation_rates.txt -parameters: - - name: start_date - default_value: 2010 - - name: end_date - default_value: 2012 -outputs: - - name: output_file - path: src/file.txt - - name: output_images - path: src/images -``` - -Don't edit the outputs section manually! - -!!! warning - - If you edit the inputs or the parameters section in the `config.yaml` file. You must return to the [steop] - diff --git a/docs/model_configuration/deprecated/09-publish.md b/docs/model_configuration/deprecated/09-publish.md deleted file mode 100644 index 9302051..0000000 --- a/docs/model_configuration/deprecated/09-publish.md +++ /dev/null @@ -1,10 +0,0 @@ -It's time to upload your code, image and model configuration. - -``` -$ mic model_configuration push -Pushing image -Pushing Git repository -Pushing Model Configuration - -URL Model Configuration -``` \ No newline at end of file diff --git a/docs/model_configuration/deprecated/docker.md b/docs/model_configuration/deprecated/docker.md deleted file mode 100644 index 32c6af8..0000000 --- a/docs/model_configuration/deprecated/docker.md +++ /dev/null @@ -1 +0,0 @@ -## \ No newline at end of file diff --git a/docs/notebooks/upload_component.md b/docs/notebooks/upload_component.md index 6af5cbf..8926c72 100644 --- a/docs/notebooks/upload_component.md +++ b/docs/notebooks/upload_component.md @@ -1,6 +1,6 @@ ## Configure your credentials -To upload your model component, you should [configure your credentials for MIC](../commands/configure.md). You do not need to setup GitHub credentials. +To upload your model component, you should [configure your credentials for MIC](../commands/configure.md). ### Upload Docker Image diff --git a/docs/overview.md b/docs/overview.md index 718904d..d0003c8 100644 --- a/docs/overview.md +++ b/docs/overview.md @@ -49,16 +49,16 @@ Use the wrapper generated in Step 7 to execute your model. **Expected outcome of this step**: A successful test run and encapsulation of your software. ### Step 9: Upload the model component software and image -MIC will upload the MIC Wrapper and your software in GitHub. MIC will also upload the component image in DockerHub and upload your software in the MINT Model Catalog, which will make it accessible by others through MINT services and interfaces to be run in their own local hosts and servers. This will give them unique identifiers that represent the snapshot of the software that you wanted to encapsulate so that any future updates to your software or your software component can be distinct from each other. +MIC will upload the MIC Wrapper and your software in the MINT storage server. MIC will also upload the component image in DockerHub and upload your software in the MINT Model Catalog, which will make it accessible by others through MINT services and interfaces to be run in their own local hosts and servers. This will give them unique identifiers that represent the snapshot of the software that you wanted to encapsulate so that any future updates to your software or your software component can be distinct from each other. -**Expected results after completing this step**: Your model component wrapper will be uploaded to Github, and your model component image to DockerHub. Both will receive a tag and will be versioned, and will be archived so they are available to anyone anywhere. MINT will have an entry for your component in the MINT Model Catalog, which will be accessible through your browser. Anyone using [DAME](dame-cli.readthedocs.io/) can run your component with their own data. Anyone using MINT can run your component with appropriate datasets available in MINT. +**Expected results after completing this step**: Your model component wrapper will be uploaded to MINT server, and your model component image to DockerHub. Both will receive a tag and will be versioned, and will be archived so they are available to anyone anywhere. MINT will have an entry for your component in the MINT Model Catalog, which will be accessible through your browser. Anyone using [DAME](dame-cli.readthedocs.io/) can run your component with their own data. Anyone using MINT can run your component with appropriate datasets available in MINT. ## Using MIC to Create Model Components The `pkg` command in MIC guides you through nine steps to create a model component that encapsulates your model code and makes it available so others can easily run it in their own local environments. To accomplish this, MIC relies on state-of-the-art virtualization tools (including Docker), software engineering best practices, and semantic reasoning for validation. -MIC creates: 1) a MIC Directory, 2) a MIC File, 3) a MIC Wrapper, 4) a component virtualization image, 5) archival versions of your model code, MIC directories and files, and image in GitHub and DockerHub, and 6) a model component entry in the MINT Model Catalog. +MIC creates: 1) a MIC Directory, 2) a MIC File, 3) a MIC Wrapper, 4) a component virtualization image, 5) archival versions of your model code, MIC directories and files, and image in the MINT Storage server and DockerHub, and 6) a model component entry in the MINT Model Catalog. Once you have MIC installed, you can type `mic pkg --help` to see the following message that lists all the steps: @@ -89,7 +89,7 @@ Commands: run Run your model component with the MIC Wrapper generated in the previous step - upload Upload your code to GitHub, DockerHub and your + upload Upload your code to DockerHub and your model component on the MINT Model Catalog. ``` diff --git a/mkdocs.yml b/mkdocs.yml index e6f7e57..7a37db3 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -4,7 +4,7 @@ nav: - Installation: installation.md - MIC Overview: overview.md - 'Instructions': - - 'Setting up Docker and GitHub': 'model_configuration/01-DockerGitHub.md' + - 'Setting up Docker': 'model_configuration/01-DockerGitHub.md' - 'Preparing your executable': 'model_configuration/02-Preparing your executable.md' - 'Using the command line': - 'Option 1: Parameters values passed through command line': diff --git a/pytest.ini b/pytest.ini index cbf838e..95b234d 100644 --- a/pytest.ini +++ b/pytest.ini @@ -3,7 +3,7 @@ testpaths = src/mic/tests norecursedirs=dist build .tox scripts src/mic/resources addopts = --basetemp=/tmp/mydir - --cov=mic - --cov-report=term + #--cov=mic + #--cov-report=term #src/mic/tests diff --git a/setup.py b/setup.py index 983b30b..8dc0b62 100644 --- a/setup.py +++ b/setup.py @@ -11,10 +11,9 @@ "Jinja2>=2.11.2", "PyYAML>=5.3.1", "modelcatalog-api==7.1.0", - "pygit2>=1.2.1", - "PyGithub>=1.43.5", "ipython2cwlmosorio==0.0.5", - "validators" + "validators", + "mint_upload" ] diff --git a/src/mic/__main__.py b/src/mic/__main__.py index 192dfaa..3685d95 100644 --- a/src/mic/__main__.py +++ b/src/mic/__main__.py @@ -1,3 +1,6 @@ +"""Main file for cmd +""" + import collections import sys from pathlib import Path @@ -16,6 +19,8 @@ @click.option("--verbose", "-v", default=0, count=True) def cli(verbose): _utils.init_logger() + if verbose: + pass try: lv = ".".join(_utils.get_latest_version().split(".")[:3]) except Exception as e: @@ -41,7 +46,7 @@ def version(debug=False): @cli.command(short_help="Configure credentials", help="Configure your credentials to access the Model Catalog API, " - "GitHub and Docker features") + "and Docker features") @click.option( "--profile", "-p", @@ -57,14 +62,11 @@ def version(debug=False): @click.option('--password', prompt="Password", required=True, hide_input=True, help="Your password") @click.option('--name', prompt='Name', help='Your name', required=True) -@click.option('--git_username', prompt='GitHub Username', help='Your GitHub Username', required=True) -@click.option('--git_token', prompt='GitHub API token', help='Your GitHub API token', required=True, hide_input=False) @click.option('--dockerhub_username', prompt='Docker Username', help='Your Docker Username') -def credentials(server, username, password, git_username, git_token, name, dockerhub_username, profile="default"): +def credentials(server, username, password, name, dockerhub_username, profile="default"): try: email = username - configure_credentials(server, username, password, git_username, git_token, name, email, dockerhub_username, - profile) + configure_credentials(server, username, password, name, email, dockerhub_username, profile) except Exception as e: click.secho("Unable to create configuration file", fg="red") diff --git a/src/mic/cli_docs.py b/src/mic/cli_docs.py index 7df6de7..d7b91c5 100644 --- a/src/mic/cli_docs.py +++ b/src/mic/cli_docs.py @@ -65,7 +65,7 @@ def info_end_run(execution_dir): click.secho("Success", fg="green") click.secho(f"You can see the result at {execution_dir}", fg="blue") click.secho("The next step is `mic pkg upload`") - click.secho("The step is going to upload the MIC Wrapper to GitHub, " + click.secho("The step is going to upload the MIC Wrapper, " "the DockerImage on DockerHub and the Model Configuration on the MINT Model Catalog") diff --git a/src/mic/click_encapsulate/commands.py b/src/mic/click_encapsulate/commands.py index 104c9de..dca1e87 100644 --- a/src/mic/click_encapsulate/commands.py +++ b/src/mic/click_encapsulate/commands.py @@ -8,10 +8,10 @@ import mic import semver from mic import _utils -from mic._utils import find_dir, get_filepaths, obtain_id, check_mic_path, make_log_file, log_system_info, \ - get_mic_logger, log_command -from mic.cli_docs import info_start_inputs, info_start_outputs, info_start_wrapper, info_end_inputs, info_end_outputs, \ - info_end_wrapper, info_start_run, info_end_run, info_end_run_failed, info_start_publish, info_end_publish, \ +from mic._utils import find_dir, get_filepaths, obtain_id, check_mic_path, make_log_file,\ + log_system_info, get_mic_logger, log_command +from mic.cli_docs import info_start_inputs, info_start_outputs, info_start_wrapper, info_end_inputs, \ + info_end_outputs, info_end_wrapper, info_start_run, info_end_run, info_end_run_failed, info_start_publish, info_end_publish, \ info_end_publish_dt from mic.component.detect import detect_framework_main, detect_new_reprozip, extract_dependencies from mic.component.executor import copy_code_to_src, compress_directory, execute_local, copy_config_to_src @@ -23,10 +23,9 @@ get_inputs, get_parameters, get_outputs_mic, get_code, add_params_from_config, get_framework from mic.constants import * from mic.publisher.docker import publish_docker, build_docker -from mic.publisher.github import push from mic.publisher.model_catalog import create_model_catalog_resource, publish_model_configuration, \ publish_data_transformation, create_data_transformation_resource - +from mic.publisher.s3 import push logging = get_mic_logger() @@ -67,8 +66,7 @@ def cli(verbose): default=None) def start(user_execution_directory, name, image): """ - This step generates a mic.yaml file and the directories (data/, src/, docker/). It also initializes a local - GitHub repository + This step generates a mic.yaml file and the directories (data/, src/, docker/). The argument: `model_configuration_name` is the name of the model component you are defining in MIC """ @@ -114,7 +112,7 @@ def start(user_execution_directory, name, image): container_name = f"{name}_{str(uuid.uuid4())[:8]}" write_spec(mic_config_path, NAME_KEY, name) write_spec(mic_config_path, DOCKER_KEY, user_image) - write_spec(mic_config_path, FRAMEWORK_KEY, framework) + write_spec(mic_config_path, FRAMEWORK_KEY, framework.label) write_spec(mic_config_path, CONTAINER_NAME_KEY, container_name) @@ -127,17 +125,9 @@ def start(user_execution_directory, name, image): if custom_image: click.secho(f""" - You are using a custom image - You must install mic and reprozip - $ pip3 install mic reprozip +You are using a custom image +Installing MIC and some dependencies """, fg="green") - else: - click.secho(f""" - You are in a Linux environment Debian distribution. - You can use `apt` to install new packages - For example: - $ apt install r-base - """, fg="green") try: os.system(docker_cmd) @@ -640,7 +630,7 @@ def run(mic_file): @cli.command( - short_help="Upload your code to GitHub, your image to DockerHub and your model component to the MINT Model Catalog.") + short_help="Upload your code, your image to DockerHub and your model component to the MINT Model Catalog.") @click.option( "-f", "--mic_file", @@ -661,7 +651,7 @@ def run(mic_file): default=None) def upload(mic_file, profile, mc, dt): """ - Upload your MIC wrapper (including all the contents of the /src folder) to GitHub, the Docker Image to DockerHub + Upload your MIC wrapper (including all the contents of the /src folder), the Docker Image to DockerHub and the model component to MINT Model Catalog. - You must pass the MIC_FILE (mic.yaml) as an argument using the (-f) option or run the diff --git a/src/mic/component/detect.py b/src/mic/component/detect.py index b6af102..5a9c11d 100644 --- a/src/mic/component/detect.py +++ b/src/mic/component/detect.py @@ -1,12 +1,12 @@ import os -import subprocess from datetime import datetime from pathlib import Path -from mic._utils import get_mic_logger import click -from mic.component.initialization import detect_framework, render_dockerfile, render_conda +from mic._utils import get_mic_logger +from mic.component.initialization import detect_framework, render_dockerfile from mic.component.python3 import freeze -from mic.constants import DOCKER_DIR, handle, Framework, REQUIREMENTS_FILE, MIC_DIR, REPRO_ZIP_TRACE_DIR +from mic.constants import DOCKER_DIR, handle, Framework, REQUIREMENTS_FILE, \ + MIC_DIR, REPRO_ZIP_TRACE_DIR logging = get_mic_logger() @@ -46,7 +46,7 @@ def detect_framework_main(user_execution_directory): click.secho("Please select the correct option") click.secho("This information allows MIC to select the correct Docker Image") - framework = click.prompt("Select a option ".format(Framework), + framework = click.prompt(f"""Select a option {Framework}""", show_choices=True, type=click.Choice(frameworks, case_sensitive=False), value_proc=handle diff --git a/src/mic/component/initialization.py b/src/mic/component/initialization.py index cbe6532..fd52805 100644 --- a/src/mic/component/initialization.py +++ b/src/mic/component/initialization.py @@ -4,7 +4,6 @@ from mic._utils import get_mic_logger from jinja2 import Environment, PackageLoader, select_autoescape from mic.constants import * -from mic.publisher.github import get_local_repo import platform env = Environment( @@ -16,6 +15,7 @@ logging = get_mic_logger() + def set_mask(value): os.umask(value) @@ -24,7 +24,8 @@ def create_base_directories(mic_component_dir: Path, interactive=True): if mic_component_dir.exists(): click.secho("The directory {} already exists. If you continue, you can lose a previous component".format( mic_component_dir.name), fg="yellow") - logging.info("When creating base dir found base directory already exists: {}".format(mic_component_dir)) + logging.info("When creating base dir found base directory already exists: {}".format( + mic_component_dir)) if interactive and not click.confirm("Do you want to continue?", default=True, show_default=True): logging.info("User aborted initialization") click.secho("Initialization aborted", fg="blue") @@ -33,7 +34,8 @@ def create_base_directories(mic_component_dir: Path, interactive=True): set_mask(0) mic_component_dir.mkdir(exist_ok=True) except Exception as e: - click.secho("Error: {} could not be created".format(mic_component_dir), fg="red") + click.secho("Error: {} could not be created".format( + mic_component_dir), fg="red") logging.exception("Could not create base dir: {}".format(mic_component_dir)) exit(1) @@ -43,13 +45,14 @@ def create_base_directories(mic_component_dir: Path, interactive=True): src.mkdir(parents=True, exist_ok=True) docker.mkdir(parents=True, exist_ok=True) data.mkdir(parents=True, exist_ok=True) - get_local_repo(mic_component_dir) logging.info("MIC has initialized the component") click.secho("MIC has initialized the component.") click.secho("[Created] {}: {}".format(DATA_DIR, mic_component_dir / DATA_DIR)) - click.secho("[Created] {}: {}".format(DOCKER_DIR, mic_component_dir / DOCKER_DIR)) + click.secho("[Created] {}: {}".format( + DOCKER_DIR, mic_component_dir / DOCKER_DIR)) click.secho("[Created] {}: {}".format(SRC_DIR, mic_component_dir / SRC_DIR)) - click.secho("[Created] {}: {}".format(CONFIG_YAML_NAME, mic_component_dir / CONFIG_YAML_NAME)) + click.secho("[Created] {}: {}".format( + CONFIG_YAML_NAME, mic_component_dir / CONFIG_YAML_NAME)) return mic_component_dir @@ -118,7 +121,8 @@ def render_run_sh(directory: Path, def render_io_sh(directory: Path, inputs: dict, parameters: dict, configs: list) -> Path: template = env.get_template(IO_FILE) file = directory / SRC_DIR / IO_FILE - if configs is None: configs = [] + if configs is None: + configs = [] list_config = [value[PATH_KEY] for key, value in configs.items()] with open(file, "w") as f: @@ -152,20 +156,20 @@ def render_dockerfile(model_directory: Path, language: Framework, custom=False) try: os = platform.system().lower() logging.debug("OS name: {}".format(os)) - except exception as e: + except Exception as e: os = "unknown" logging.debug("OS name: {}".format(os)) logging.warning("Error while detecting os: {}".format(e)) with open(run_file, "w") as f: - content = render_template(template=template, language=language, custom=custom, os=os) + content = render_template( + template=template, language=language, custom=custom, os=os) if os == "windows": logging.info("Windows os detected. Converting line endings") f.write(content) - entrypoint_file = model_directory / DOCKER_DIR / ENTRYPOINT_FILE template = env.get_template(ENTRYPOINT_FILE) with open(entrypoint_file, "w") as f: @@ -175,6 +179,7 @@ def render_dockerfile(model_directory: Path, language: Framework, custom=False) return run_file + def recursive_convert_to_lf(dir): """ Convert windows CRLF endings into unix LF endigns. Returns an array of converted files @@ -187,7 +192,7 @@ def recursive_convert_to_lf(dir): if root.find(".git") == -1: with open(Path(root) / Path(file), 'rb') as open_file: content = open_file.read() - + content = content.replace(b'\r\n', b'\n') converted_files.append("{}".format(file)) diff --git a/src/mic/config_yaml.py b/src/mic/config_yaml.py index 2e42171..18c124f 100644 --- a/src/mic/config_yaml.py +++ b/src/mic/config_yaml.py @@ -267,7 +267,7 @@ def add_params_from_config(yaml_path: Path, config_path: Path): if added: click.secho("Default values will need to be added in {} for each parameter".format(CONFIG_YAML_NAME),fg="green") -def get_inputs_parameters(config_yaml_path: Path) -> (dict, dict, dict): +def get_inputs_parameters(config_yaml_path: Path): inputs = get_inputs(config_yaml_path) parameters = get_parameters(config_yaml_path) outputs = get_outputs_mic(config_yaml_path) diff --git a/src/mic/constants.py b/src/mic/constants.py index d3dd35a..3e27b8e 100644 --- a/src/mic/constants.py +++ b/src/mic/constants.py @@ -43,12 +43,9 @@ EXECUTIONS_DIR = "executions" TOTAL_STEPS = 8 MINT_COMPONENT_ZIP = "mint_component" -GIT_TOKEN_KEY = "git_token" -GIT_USERNAME_KEY = "git_username" DOCKER_KEY = "docker_image" FRAMEWORK_KEY = "framework" LAST_EXECUTION_DIR = "last_execution_dir" -REPO_KEY = "github_repo_url" VERSION_KEY = "version" DOCKER_USERNAME_KEY = "dockerhub_username" MINT_COMPONENT_KEY = "mint_component_url" @@ -68,7 +65,7 @@ MAP_PYTHON_MODEL_CATALOG = {"str": "string", "bool": "boolean", "int": "int", "float": "float"} DEFAULT_PARAMETER_COMMENT = "# value added by MIC. Replace with your own default value" DEFAULT_DESCRIPTION_MESSAGE = "# insert description left of this comment" -EXECUTABLE_EXTENSIONS = [".sh", ".py", ".jar", ".R", ".m", ".cpp", ".c", ".exe", ".bat", ".js", ".php", ".cs", ".pl", ".vb"] +EXECUTABLE_EXTENSIONS = [".sh", ".py", ".jar", ".r", ".m", ".cpp", ".c", ".exe", ".bat", ".js", ".php", ".cs", ".pl", ".vb"] LOG_FILE = "mic.log" # Default output messages @@ -82,7 +79,7 @@ class Framework(Enum): PYTHON38 = ("python3.8", "mintproject/python:3.8", ".py") CONDA = ("conda4.7.12", "mintproject/conda:4.7.12", ".py") JAVA = ("java8", "mintproject/java:8", ".jar") - GENERIC = ("general", "mintproject/generic:latest") + GENERIC = ("generic", "mintproject/generic:latest", "generic") def __init__(self, label, image, extension=None): self.label = label diff --git a/src/mic/credentials.py b/src/mic/credentials.py index 28a2e13..fc64456 100644 --- a/src/mic/credentials.py +++ b/src/mic/credentials.py @@ -22,7 +22,7 @@ def get_credentials(profile: str) -> dict: raise ValueError("Profile doesn't exists") -def configure_credentials(server, username, password, git_username, git_token, name, email, dockerhub_username, profile): +def configure_credentials(server, username, password, name, email, dockerhub_username, profile): credentials_file = pathlib.Path( os.getenv("MINT_CREDENTIALS_FILE", __DEFAULT_MINT_API_CREDENTIALS_FILE__) ).expanduser() @@ -38,8 +38,6 @@ def configure_credentials(server, username, password, git_username, git_token, n "server": server, "username": username, "password": password, - "git_username": git_username, - "git_token": git_token, "name": name, "email": email, "dockerhub_username": dockerhub_username, @@ -90,17 +88,7 @@ def print_list_credentials(profile, short): if not short: for field in prof: # Dont print password or token - if field != "password" and field != "git_token": + if field != "password": click.secho(" {}".format(field), nl=False, fg="green") click.secho(": {}".format(prof[field])) - # Dont print full token for security reasons - elif field == "git_token": - # Its safe to print if its obviously not a github token - if len(prof[field]) < 6: - click.secho(" {}".format(field), nl=False, fg="green") - click.secho(": {}".format(prof[field])) - - else: - click.secho(" {}".format(field), nl=False, fg="green") - click.secho(": Ending in \"...{}\"".format((prof[field])[-5:])) click.echo("\n") diff --git a/src/mic/publisher/github.py b/src/mic/publisher/github.py deleted file mode 100644 index f34811b..0000000 --- a/src/mic/publisher/github.py +++ /dev/null @@ -1,365 +0,0 @@ -import datetime -import logging -import os -import re -import shutil -from pathlib import Path - -import click -import pygit2 as pygit2 -import semver -from distutils.version import StrictVersion -from github import Github -from mic._utils import get_mic_logger -from mic.config_yaml import write_spec -from mic.constants import MINT_COMPONENT_ZIP, GIT_TOKEN_KEY, GIT_USERNAME_KEY, SRC_DIR, REPO_KEY, VERSION_KEY, \ - MINT_COMPONENT_KEY, DEFAULT_CONFIGURATION_WARNING -from mic.credentials import get_credentials - -logging = get_mic_logger() -author = pygit2.Signature('MIC Bot', 'bot@mint.isi.edu') - -def push_cwl(model_directory: Path, mic_config_path: Path, name: str, profile): - repository_name = name - click.secho("Creating the git repository") - logging.info("Creating the git repository") - repo = get_local_repo(model_directory) - logging.info("Compressing code") - logging.info("Creating a new commit") - click.secho("Creating a new commit") - git_commit(repo) - click.secho("Creating or using the GitHub repository") - url = check_create_remote_repo(repo, profile, repository_name) - repository_name = url.split('/')[-1].replace(".git", "") - write_spec(mic_config_path, REPO_KEY, url) - logging.info("Creating a new version") - click.secho("Creating a new version") - _version = git_tag(repo, author) - - logging.info("Pushing changes to the server") - click.secho("Pushing your changes to the server") - remote = repo.remotes["origin"] - try: - git_pull(repo, remote) - except AssertionError as e: - logging.error("Unable to handle git conflict, user must manually fix") - click.secho("Unable to handle git conflict, please fix them manually", fg="red") - exit(1) - - git_push(repo, profile, _version) - write_spec(mic_config_path, VERSION_KEY, _version) - - repo = get_github_repo(profile, repository_name) - file = None - for i in repo.get_contents(""): - if i.name == "{}.cwl".format(MINT_COMPONENT_ZIP): - file = i - write_spec(mic_config_path, MINT_COMPONENT_KEY, file.download_url) - break - if not file: - click.secho(f"Mint component not found {MINT_COMPONENT_ZIP}.cwl", fg="red") - logging.error("Could not find mint component: {}.zip".format(MINT_COMPONENT_ZIP)) - exit(1) - - logging.info("Push complete: {}".format({'repository': url, 'version': _version})) - click.secho("Repository: {}".format(url)) - click.secho("Version: {}".format(_version)) - - - -def push(model_directory: Path, mic_config_path: Path, name: str, profile): - repository_name = name - click.secho("Creating the git repository") - logging.info("Creating the git repository") - repo = get_local_repo(model_directory) - logging.info("Compressing code") - click.secho("Compressing your code") - compress_src_dir(model_directory) - logging.info("Creating a new commit") - click.secho("Creating a new commit") - git_commit(repo) - click.secho("Creating or using the GitHub repository") - url = check_create_remote_repo(repo, profile, repository_name) - repository_name = url.split('/')[-1].replace(".git", "") - write_spec(mic_config_path, REPO_KEY, url) - logging.info("Creating a new version") - click.secho("Creating a new version") - _version = git_tag(repo, author) - - logging.info("Pushing changes to the server") - click.secho("Pushing your changes to the server") - remote = repo.remotes["origin"] - try: - git_pull(repo, remote) - except AssertionError as e: - logging.error("Unable to handle git conflict, user must manually fix") - click.secho("Unable to handle git conflict, please fix them manually", fg="red") - exit(1) - - git_push(repo, profile, _version) - write_spec(mic_config_path, VERSION_KEY, _version) - - repo = get_github_repo(profile, repository_name) - file = None - for i in repo.get_contents(""): - if i.name == "{}.zip".format(MINT_COMPONENT_ZIP): - file = i - write_spec(mic_config_path, MINT_COMPONENT_KEY, file.download_url) - break - if not file: - click.secho(f"Mint component not found {MINT_COMPONENT_ZIP}.zip", fg="red") - logging.error("Could not find mint component: {}.zip".format(MINT_COMPONENT_ZIP)) - exit(1) - - logging.info("Push complete: {}".format({'repository': url, 'version': _version})) - click.secho("Repository: {}".format(url)) - click.secho("Version: {}".format(_version)) - - -def git_commit(repo): - repo.index.add_all() - repo.index.write() - tree = repo.index.write_tree() - parent = None - try: - parent = repo.revparse_single('HEAD') - except KeyError: - pass - - parents = [] - if parent: - parents.append(parent.oid.hex) - repo.create_commit('refs/heads/master', author, author, "automated mic", tree, parents) - - -def get_local_repo(model_path: Path): - return pygit2.Repository(pygit2.discover_repository(model_path)) if pygit2.discover_repository( - model_path) else pygit2.init_repository( - model_path, False) - - -def compress_src_dir(model_path: Path): - """ - Compress the directory src and create a zip file - """ - zip_file_name = model_path / MINT_COMPONENT_ZIP - src_dir = model_path / SRC_DIR - mic_component_path = model_path / f"{MINT_COMPONENT_ZIP}.zip" - if mic_component_path.exists(): - os.remove(mic_component_path) - zip_file_path = shutil.make_archive(zip_file_name.name, 'zip', root_dir=model_path.parent, - base_dir=src_dir.relative_to(model_path.parent)) - shutil.move(zip_file_path, mic_component_path) - return zip_file_path - - -def check_create_remote_repo(repo, profile, model_name): - if "origin" in [i.name for i in repo.remotes]: - origin__url = repo.remotes["origin"].url - - try: - github_repo_exists(model_name, profile) - except Exception: - click.secho(f"The git repository has a remote server configured {origin__url}, " - f"but it does not exist on github", fg="red") - click.echo("You can delete the reference to the repository running\n$ git remote remove origin") - exit(1) - - click.secho(f"The git repository has a remote server configured {origin__url}") - return origin__url - else: - click.secho(f"The git repository has not a remote server configured ") - click.secho(f"Creating a new repository on GitHub") - repo_github = github_create_repo(profile, model_name) - url = repo_github.clone_url - repo.remotes.create("origin", url) - git_push(repo, profile) - click.secho(f"The url is: {url}") - return url - - -def github_repo_exists(model_name, profile): - g = github_auth(profile) - try: - g.get_user().get_repo(model_name) - except Exception as e: - raise e - - -def github_auth(profile): - git_token, git_username = github_config(profile) - g = Github(git_username, git_token) - github_login(g) - return g - - -def get_github_repo(profile, model_name): - git_token, git_username = github_config(profile) - g = Github(git_username, git_token) - github_login(g) - user = g.get_user() - return user.get_repo(model_name) - - -def git_add_remote(repo, url): - repo.remotes.create("origin", url) - - -def git_pull(repo, remote, branch="master"): - remote.fetch() - remote_master_id = repo.lookup_reference('refs/remotes/origin/%s' % (branch)).target - merge_result, _ = repo.merge_analysis(remote_master_id) - # Up to date, do nothing - if merge_result & pygit2.GIT_MERGE_ANALYSIS_UP_TO_DATE: - return True - elif merge_result & pygit2.GIT_MERGE_ANALYSIS_FASTFORWARD: - repo.checkout_tree(repo.get(remote_master_id)) - try: - master_ref = repo.lookup_reference('refs/heads/%s' % (branch)) - master_ref.set_target(remote_master_id) - except KeyError: - repo.create_branch(branch, repo.get(remote_master_id)) - repo.head.set_target(remote_master_id) - elif merge_result & pygit2.GIT_MERGE_ANALYSIS_NORMAL: - repo.merge(remote_master_id) - - if repo.index.conflicts is not None: - for conflict in repo.index.conflicts: - click.echo(f"Conflicts found in: {conflict[0].path}") - logging.debug(f"Conflicts found in: {conflict[0].path}") - raise AssertionError('Conflicts') - - user = repo.default_signature - tree = repo.index.write_tree() - commit = repo.create_commit('HEAD', - user, - user, - 'Merge', - tree, - [repo.head.target, remote_master_id]) - # We need to do this or git CLI will think we are still merging. - repo.state_cleanup() - else: - logging.warning("Unknown merge analysis result") - raise AssertionError('Unknown merge analysis result') - - -def git_push(repo, profile, tag=None): - git_token, git_username = github_config(profile) - callbacks = pygit2.RemoteCallbacks(pygit2.UserPass(git_token, 'x-oauth-basic')) - remote = repo.remotes["origin"] - remote.push(['refs/heads/master'], callbacks=callbacks) - if tag: - remote.push(['refs/tags/{}'.format(tag)], callbacks=callbacks) - - -def git_tag(repo, tagger): - """ - If there is a release, increment the version. - """ - version = get_next_tag(repo) - repo.create_tag(str(version), - repo.revparse_single('HEAD').oid.hex, - pygit2.GIT_OBJ_COMMIT, - tagger, - str(version)) - - click.secho("New version: {}".format(str(version))) - logging.debug(f"New version: {str(version)}") - return str(version) - - -def get_next_tag(repo): - regex = re.compile('^refs/tags') - _tags = filter(lambda r: regex.match(r), repo.listall_references()) - tags = [tag.split('/')[-1] for tag in _tags] - tags.sort(key=StrictVersion) - today = datetime.date.today() - version_today = semver.VersionInfo.parse("{}.{}.{}".format(int(today.year) % 100, today.month, 1)) - if tags: - version_str = tags[-1] - try: - version = semver.VersionInfo.parse(version_str) - if int(version.minor) != today.month or int(version.major) != today.year % 100: - return version_today - else: - return version.bump_patch() - except ValueError as e: - logging.info(e) - pass - return version_today - - -def github_create_repo(profile, model_name): - """ - Upload the directory to git - If the directory is not a git directory, create it - If the git directory doesn't have a remote origin, create a github repository - @type profile: str - @param profile: the profile to use in the credentials file - @type: directory: Path - """ - git_token, git_username = github_config(profile) - g = Github(git_username, git_token) - github_login(g) - user = g.get_user() - repo = None - try: - repo = user.get_repo(model_name) - except: - # TODO: github.GithubException.UnknownObjectException: 404 - # {"message": "Not Found", "documentation_url": "https://developer.github.com/v3/repos/#get"} - pass - if repo: - logging.info("Selected repository already exists") - if not click.confirm("The repo {} exists. Do you want to use it?".format(model_name), default=True): - click.secho("Please rename the directory", fg="green") - logging.info("User has aborted") - exit(0) - else: - repo = user.create_repo(model_name) - return repo - - -def remove_temp_files(model_path: Path): - component_folder = model_path / "{}_component".format(model_path.name) - zip_folder = model_path / "{}.zip".format(MINT_COMPONENT_ZIP) - try: - if component_folder.exists(): - shutil.rmtree(component_folder) - - if zip_folder.exists(): - os.remove(zip_folder) - - except Exception as e: - logging.warning("Could not remove temporary files: {}".format(e)) - click.secho("Warning: error when removing temporary files", fg="yellow") - - -def github_config(profile): - # Try to get git username and token from credentials file - try: - credentials = get_credentials(profile) - git_username = credentials[GIT_USERNAME_KEY] - git_token = credentials[GIT_TOKEN_KEY] - except KeyError: - click.secho(DEFAULT_CONFIGURATION_WARNING + " {}".format(profile), fg="yellow") - click.error(DEFAULT_CONFIGURATION_WARNING + " {}".format(profile)) - exit(1) - return git_token, git_username - - -def github_login(g, debug=False): - try: - if g.get_user().login is None: - click.secho("User profile GitHub credentials are invalid. Please enter a valid token and username") - logging.error("User profile GitHub credentials are invalid. Please enter a valid token and username") - exit(1) - # I know its bad to except Exception but it doesnt catch it when I except TypeError, and the only way this *should* - # fail is if the credentials are bad so... - except Exception as e: - click.secho("User profile GitHub credentials are invalid. Please enter a valid token and username") - logging.error("User profile GitHub credentials are invalid. Please enter a valid token and username") - if debug: - click.secho(e, fg="yellow") - exit(1) diff --git a/src/mic/publisher/model_catalog.py b/src/mic/publisher/model_catalog.py index d748ad5..d1ef8a4 100644 --- a/src/mic/publisher/model_catalog.py +++ b/src/mic/publisher/model_catalog.py @@ -86,6 +86,7 @@ def create_model_catalog_resource(mint_config_file, name=None, execution_dir=Non if code is None: click.secho("Failed to upload. Missing information zip file", fg="red") + exit(0) else: model_configuration.has_component_location = [code] return model_configuration diff --git a/src/mic/publisher/s3.py b/src/mic/publisher/s3.py new file mode 100644 index 0000000..610cd9f --- /dev/null +++ b/src/mic/publisher/s3.py @@ -0,0 +1,61 @@ +"""Command to upload +""" +import logging +import os +import shutil +from pathlib import Path +from datetime import datetime +import click +from mic.credentials import get_credentials +from mint_upload.object_storage import Uploader +from mic.config_yaml import write_spec +from mic.constants import VERSION_KEY, MINT_COMPONENT_KEY, MINT_COMPONENT_ZIP, SRC_DIR + +def new_version(): + return datetime.now().strftime("%Y%m%d-%H%M%S") + +def push(model_directory: Path, mic_config_path: Path, name: str, profile): + repository_name = name + _version = new_version() + write_spec(mic_config_path, VERSION_KEY, _version) + logging.info("Compressing code") + click.secho("Compressing your code") + zip_file = compress_src_dir(model_directory, _version) + url = upload_file(zip_file, profile, "components") + write_spec(mic_config_path, MINT_COMPONENT_KEY, url) + logging.info("Push complete: {}".format({'repository': url, 'version': _version})) + click.secho("Repository: {}".format(url)) + click.secho("Version: {}".format(_version)) + +def upload_file(file_name: Path, profile, bucket_name): + mint_auth_server = "https://auth.mint.mosorio.dev/auth/realms/development/protocol/openid-connect/token" + mint_s3_server = "https://s3.mint.mosorio.dev" + credentials = get_credentials(profile) + username = credentials["username"] + password = credentials["password"] + uploader = Uploader(mint_s3_server, mint_auth_server, username, password) + try: + uploader.upload_file( + str(file_name), + bucket_name + ) + except Exception as error: + logging.error("Unable to upload", exc_info=True) + exit(0) + return f"""{mint_s3_server}/{bucket_name}/{file_name.name}""" + + +def compress_src_dir(model_path: Path, version: str) -> Path: + """ + Compress the directory src and create a zip file + """ + name = f"""{MINT_COMPONENT_ZIP}_{version}""" + zip_file_name = model_path / name + src_dir = model_path / SRC_DIR + mic_component_path = model_path / f"{name}.zip" + if mic_component_path.exists(): + os.remove(mic_component_path) + zip_file_path = shutil.make_archive(zip_file_name.name, 'zip', root_dir=model_path.parent, + base_dir=src_dir.relative_to(model_path.parent)) + shutil.move(zip_file_path, mic_component_path) + return Path(mic_component_path) \ No newline at end of file diff --git a/src/mic/templates/Dockerfile b/src/mic/templates/Dockerfile index 911ded2..2f208b2 100644 --- a/src/mic/templates/Dockerfile +++ b/src/mic/templates/Dockerfile @@ -14,6 +14,7 @@ RUN sed -i 's/\r//' set_umask.sh {% endif -%} {% endif -%} +ADD https://data.mint.isi.edu/files/install.sh /bin/mic_install.sh RUN chmod +x /set_umask.sh ENTRYPOINT ["/set_umask.sh"] diff --git a/src/mic/templates/entrypoint.sh b/src/mic/templates/entrypoint.sh index fc56448..505ec94 100644 --- a/src/mic/templates/entrypoint.sh +++ b/src/mic/templates/entrypoint.sh @@ -1,3 +1,5 @@ #!/bin/bash umask 0000 +bash /bin/mic_install.sh +find . -type f -print0 | xargs -0 dos2unix &> /dev/null /bin/bash diff --git a/src/mic/templates/install.sh b/src/mic/templates/install.sh new file mode 100644 index 0000000..e19cf3a --- /dev/null +++ b/src/mic/templates/install.sh @@ -0,0 +1,375 @@ +#!/bin/sh +set -e +# MIC for Linux installation script +# +# +# This script is meant for quick & easy install via: +# $ curl -fsSL https://get.mic.com -o get-mic.sh +# $ sh get-mic.sh +# +# For test builds (ie. release candidates): +# $ curl -fsSL https://test.mic.com -o test-mic.sh +# $ sh test-mic.sh +# +# NOTE: Make sure to verify the contents of the script +# you downloaded matches the contents of install.sh +# located at https://github.com/mic/mic-install +# before executing. +# +# Git commit from https://github.com/mic/mic-install when +# the script was uploaded (Should only be modified by upload job): + +command_exists() { + command -v "$@" > /dev/null 2>&1 +} + + +DRY_RUN=${DRY_RUN:-} +while [ $# -gt 0 ]; do + case "$1" in + --dry-run) + DRY_RUN=1 + ;; + --*) + echo "Illegal option $1" + ;; + esac + shift $(( $# > 0 ? 1 : 0 )) +done + + + +is_dry_run() { + if [ -z "$DRY_RUN" ]; then + return 1 + else + return 0 + fi +} + +is_wsl() { + case "$(uname -r)" in + *microsoft* ) true ;; # WSL 2 + *Microsoft* ) true ;; # WSL 1 + * ) false;; + esac +} + +is_darwin() { + case "$(uname -s)" in + *darwin* ) true ;; + *Darwin* ) true ;; + * ) false;; + esac +} + +deprecation_notice() { + distro=$1 + date=$2 + echo + echo "DEPRECATION WARNING:" + echo " The distribution, $distro, will no longer be supported in this script as of $date." + echo " If you feel this is a mistake please submit an issue at https://github.com/mic/mic-install/issues/new" + echo + sleep 10 +} + +get_distribution() { + lsb_dist="" + # Every system that we officially support has /etc/os-release + if [ -r /etc/os-release ]; then + lsb_dist="$(. /etc/os-release && echo "$ID")" + fi + # Returning an empty string here should be alright since the + # case statements don't act unless you provide an actual value + echo "$lsb_dist" +} + +add_debian_backport_repo() { + debian_version="$1" + backports="deb http://ftp.debian.org/debian $debian_version-backports main" + if ! grep -Fxq "$backports" /etc/apt/sources.list; then + (set -x; $sh_c "echo \"$backports\" >> /etc/apt/sources.list") + fi +} + +echo_mic_as_nonroot() { + if is_dry_run; then + return + fi + if command_exists mic && [ -e /var/run/mic.sock ]; then + ( + set -x + $sh_c 'mic version' + ) || true + fi + + # intentionally mixed spaces and tabs here -- tabs are stripped by "<<-EOF", spaces are kept in the output + echo + echo "================================================================================" + echo + if [ -n "$has_rootless_extras" ]; then + echo "To run Docker as a non-privileged user, consider setting up the" + echo "Docker daemon in rootless mode for your user:" + echo + echo " micd-rootless-setuptool.sh install" + echo + echo "Visit https://docs.mic.com/go/rootless/ to learn about rootless mode." + echo + fi + echo + echo "To run the Docker daemon as a fully privileged service, but granting non-root" + echo "users access, refer to https://docs.mic.com/go/daemon-access/" + echo + echo "WARNING: Access to the remote API on a privileged Docker daemon is equivalent" + echo " to root access on the host. Refer to the 'Docker daemon attack surface'" + echo " documentation for details: https://docs.mic.com/go/attack-surface/" + echo + echo "================================================================================" + echo +} + +# Check if this is a forked Linux distro +check_forked() { + + # Check for lsb_release command existence, it usually exists in forked distros + if command_exists lsb_release; then + # Check if the `-u` option is supported + set +e + lsb_release -a -u > /dev/null 2>&1 + lsb_release_exit_code=$? + set -e + + # Check if the command has exited successfully, it means we're in a forked distro + if [ "$lsb_release_exit_code" = "0" ]; then + # Print info about current distro + cat <<-EOF + You're using '$lsb_dist' version '$dist_version'. + EOF + + # Get the upstream release info + lsb_dist=$(lsb_release -a -u 2>&1 | tr '[:upper:]' '[:lower:]' | grep -E 'id' | cut -d ':' -f 2 | tr -d '[:space:]') + dist_version=$(lsb_release -a -u 2>&1 | tr '[:upper:]' '[:lower:]' | grep -E 'codename' | cut -d ':' -f 2 | tr -d '[:space:]') + + # Print info about upstream distro + cat <<-EOF + Upstream release is '$lsb_dist' version '$dist_version'. + EOF + else + if [ -r /etc/debian_version ] && [ "$lsb_dist" != "ubuntu" ] && [ "$lsb_dist" != "raspbian" ]; then + if [ "$lsb_dist" = "osmc" ]; then + # OSMC runs Raspbian + lsb_dist=raspbian + else + # We're Debian and don't even know it! + lsb_dist=debian + fi + dist_version="$(sed 's/\/.*//' /etc/debian_version | sed 's/\..*//')" + case "$dist_version" in + 10) + dist_version="buster" + ;; + 9) + dist_version="stretch" + ;; + 8|'Kali Linux 2') + dist_version="jessie" + ;; + esac + fi + fi + fi +} + +semverParse() { + major="${1%%.*}" + minor="${1#$major.}" + minor="${minor%%.*}" + patch="${1#$major.$minor.}" + patch="${patch%%[-.]*}" +} + +do_install() { + echo "# Executing mic install script, commit: $SCRIPT_COMMIT_SHA" + + if command_exists mic; then + mic_version="$(mic version | cut -d ' ' -f2 | cut -d 'v' -f2)" + MAJOR_W=2 + MINOR_W=0 + + semverParse "$mic_version" + + shouldWarn=0 + if [ "$major" -lt "$MAJOR_W" ]; then + shouldWarn=1 + fi + + if [ "$major" -le "$MAJOR_W" ] && [ "$minor" -lt "$MINOR_W" ]; then + shouldWarn=1 + fi + + cat >&2 <<-'EOF' + Warning: the "mic" command appears to already exist on this system. + + If you already have Docker installed, this script can cause trouble, which is + why we're displaying this warning and provide the opportunity to cancel the + installation. + + If you installed the current Docker package using this script and are using it + EOF + + if [ $shouldWarn -eq 1 ]; then + cat >&2 <<-'EOF' + again to update Docker, we urge you to migrate your image store before upgrading + to v1.10+. + + You can find instructions for this here: + https://github.com/mic/mic/wiki/Engine-v1.10.0-content-addressability-migration + EOF + else + cat >&2 <<-'EOF' + again to update Docker, you can safely ignore this message. + EOF + fi + + cat >&2 <<-'EOF' + + You may press Ctrl+C now to abort this script. + EOF + ( set -x; sleep 1 ) + fi + + user="$(id -un 2>/dev/null || true)" + + sh_c='sh -c' + if [ "$user" != 'root' ]; then + if command_exists sudo; then + sh_c='sudo -E sh -c' + elif command_exists su; then + sh_c='su -c' + else + cat >&2 <<-'EOF' + Error: this installer needs the ability to run commands as root. + We are unable to find either "sudo" or "su" available to make this happen. + EOF + exit 1 + fi + fi + + if is_dry_run; then + sh_c="echo" + fi + + # perform some very rudimentary platform detection + lsb_dist=$( get_distribution ) + lsb_dist="$(echo "$lsb_dist" | tr '[:upper:]' '[:lower:]')" + + if is_wsl; then + echo + echo "WSL DETECTED: We recommend using Docker Desktop for Windows" + echo "Please get Docker Desktop from https://www.mic.com/products/mic-desktop" + echo + cat >&2 <<-'EOF' + + You may press Ctrl+C now to abort this script. + EOF + ( set -x; sleep 20 ) + fi + + case "$lsb_dist" in + + ubuntu) + if command_exists lsb_release; then + dist_version="$(lsb_release --codename | cut -f2)" + fi + if [ -z "$dist_version" ] && [ -r /etc/lsb-release ]; then + dist_version="$(. /etc/lsb-release && echo "$DISTRIB_CODENAME")" + fi + ;; + + debian|raspbian) + dist_version="$(sed 's/\/.*//' /etc/debian_version | sed 's/\..*//')" + case "$dist_version" in + 10) + dist_version="buster" + ;; + 9) + dist_version="stretch" + ;; + 8) + dist_version="jessie" + ;; + esac + ;; + + centos|rhel) + if [ -z "$dist_version" ] && [ -r /etc/os-release ]; then + dist_version="$(. /etc/os-release && echo "$VERSION_ID")" + fi + ;; + + *) + if command_exists lsb_release; then + dist_version="$(lsb_release --release | cut -f2)" + fi + if [ -z "$dist_version" ] && [ -r /etc/os-release ]; then + dist_version="$(. /etc/os-release && echo "$VERSION_ID")" + fi + ;; + + esac + + # Check if this is a forked Linux distro + check_forked + + # Run setup for each distro accordingly + case "$lsb_dist" in + ubuntu|debian|raspbian) + ( + if ! is_dry_run; then + set -x + fi + $sh_c 'apt-get update -qq >/dev/null' + $sh_c 'DEBIAN_FRONTEND=noninteractive apt-get install --no-install-recommends -y python3 python3-pip python3-dev git dos2unix > /dev/null' + echo "Installing MIC" + $sh_c 'pip3 install reprozip mic > /dev/null' + set -x + echo + echo "You are in a Linux environment Debian distribution." + echo "You can use apt to install new packages." + echo "MIC is going to push new changes with these changes." + set +x + ) + exit 0 + ;; + centos|fedora|rhel) + ( + if ! is_dry_run; then + set -x + fi + # install the correct cli version first + $sh_c 'yum install -y python3 python3-pip pytho3-devel dos2unix' + ) + exit 0 + ;; + *) + if [ -z "$lsb_dist" ]; then + if is_darwin; then + echo + echo "ERROR: Unsupported operating system 'macOS'" + echo + exit 1 + fi + fi + echo + echo "ERROR: Unsupported distribution '$lsb_dist'" + echo + exit 1 + ;; + esac + exit 1 +} + +# wrapped up in a function so that we have some protection against only getting +# half the file during "curl | sh" +do_install diff --git a/src/mic/tests/resources/254/mic/mic3.yaml b/src/mic/tests/resources/254/mic/mic3.yaml index 1782dbe..d86387d 100644 --- a/src/mic/tests/resources/254/mic/mic3.yaml +++ b/src/mic/tests/resources/254/mic/mic3.yaml @@ -1,10 +1,7 @@ step: 1 name: parameter-test docker_image: parameter-test:latest -framework: !!python/object/apply:mic.constants.Framework -- !!python/tuple - - general - - mintproject/generic:latest +framework: generic inputs: a_txt: path: a.txt diff --git a/src/mic/tests/resources/swat/mic/mic.yaml b/src/mic/tests/resources/swat/mic/mic.yaml index 530b810..9dd44f9 100644 --- a/src/mic/tests/resources/swat/mic/mic.yaml +++ b/src/mic/tests/resources/swat/mic/mic.yaml @@ -1,10 +1,7 @@ step: 2 name: swat_forecast docker_image: mosorio/swat_forecast:20.6.10 -framework: !!python/object/apply:mic.constants.Framework -- !!python/tuple - - general - - mintproject/generic:latest +framework: generic parameters: SFTMP: default_value: 0.0 diff --git a/src/mic/tests/resources/topoflow_dt/mic/mic.yaml b/src/mic/tests/resources/topoflow_dt/mic/mic.yaml index d09f327..e7c3119 100644 --- a/src/mic/tests/resources/topoflow_dt/mic/mic.yaml +++ b/src/mic/tests/resources/topoflow_dt/mic/mic.yaml @@ -1,10 +1,7 @@ step: 1 name: topoflow_dt docker_image: mintproject/dt_dame:latest -framework: !!python/object/apply:mic.constants.Framework -- !!python/tuple - - general - - mintproject/generic:latest +framework: generic parameters: data_set_id: default_value: 5babae3f-c468-4e01-862e-8b201468e3b5 diff --git a/src/mic/tests/test_model_catalog_utils.py b/src/mic/tests/test_model_catalog_utils.py index 644c4e9..6dd3faa 100644 --- a/src/mic/tests/test_model_catalog_utils.py +++ b/src/mic/tests/test_model_catalog_utils.py @@ -11,8 +11,6 @@ def test_configure(): model_catalog_password = os.environ['MODEL_CATALOG_PASSWORD_TEST'] result = runner.invoke(credentials, [ "--username", model_catalog_username, - "--git_username", "mintbot", - "--git_token", "asdfsafs", "--password", model_catalog_password, "--dockerhub_username", "a", "--name", "pedro", @@ -21,19 +19,17 @@ def test_configure(): assert result.exit_code == 0 -def test_get_api(): - runner = CliRunner() - model_catalog_username = os.environ['MODEL_CATALOG_USERNAME_TEST'] - model_catalog_password = os.environ['MODEL_CATALOG_PASSWORD_TEST'] - result = runner.invoke(credentials, [ - "--username", model_catalog_username, - "--git_username", "mintbot", - "--git_token", "asdfsafs", - "--password", model_catalog_password, - "--dockerhub_username", "a", - "--name", "pedro", - "--profile", "testing" - ]) - assert result.exit_code == 0 - api = get_api(profile="testing") - assert api +# def test_get_api(): +# runner = CliRunner() +# model_catalog_username = os.environ['MODEL_CATALOG_USERNAME_TEST'] +# model_catalog_password = os.environ['MODEL_CATALOG_PASSWORD_TEST'] +# result = runner.invoke(credentials, [ +# "--username", model_catalog_username, +# "--password", model_catalog_password, +# "--dockerhub_username", "a", +# "--name", "pedro", +# "--profile", "testing" +# ]) +# assert result.exit_code == 0 +# api = get_api(profile="testing") +# assert api