Merge pull request #232 from mintproject/develop

Release v1.0.1

src/mic/click_encapsulate/commands.py

@@ -1,21 +1,21 @@
-import ast
+import os
 import os
 import shutil
 from datetime import datetime
 from pathlib import Path
 
-import click
 import mic
 import semver
 from mic import _utils
-from mic._utils import find_dir, get_filepaths, obtain_id, recursive_mic_search, check_mic_path
+from mic._utils import find_dir, get_filepaths, obtain_id, check_mic_path
 from mic.cli_docs import info_start_inputs, info_start_outputs, info_start_wrapper, info_end_inputs, info_end_outputs, \
     info_end_wrapper, info_start_run, info_end_run, info_end_run_failed, info_start_publish, info_end_publish
 from mic.component.detect import detect_framework_main, detect_new_reprozip, extract_dependencies
 from mic.component.executor import copy_code_to_src, compress_directory, execute_local, copy_config_to_src
 from mic.component.initialization import render_run_sh, render_io_sh, render_output, create_base_directories, \
-    render_bash_color
-from mic.component.reprozip import get_inputs_outputs_reprozip, get_outputs_reprozip, relative, generate_runner, generate_pre_runner, \
+    render_bash_color, render_gitignore
+from mic.component.reprozip import get_inputs_outputs_reprozip, get_outputs_reprozip, relative, generate_runner, \
+    generate_pre_runner, \
     find_code_files
 from mic.config_yaml import write_spec, write_to_yaml, get_spec, get_key_spec, create_config_file_yaml, get_configs, \
     get_inputs, get_parameters, get_outputs_mic, get_code, add_params_from_config, get_framework
@@ -48,7 +48,7 @@ def cli(verbose):
         )
 
 
-@cli.command(short_help="Create a Linux environment to run your model. The working directory selected must"                                           
+@cli.command(short_help="Create a Linux environment to run your model. The working directory selected must"
                         " contain all the files required for the execution of your model")
 @click.argument(
     "user_execution_directory",
@@ -57,7 +57,10 @@ def cli(verbose):
     required=True
 )
 @click.option('--name', prompt="Model component name", help="Name of the model component you want for your model")
-def start(user_execution_directory, name):
+@click.option('--image',
+              help="(Optional) If you have a DockerImage, you can use it",
+              default=None)
+def start(user_execution_directory, name, image):
     """
     This step generates a mic.yaml file and the directories (data/, src/, docker/). It also initializes a local
     GitHub repository
@@ -67,13 +70,17 @@ def start(user_execution_directory, name):
     user_execution_directory = Path(user_execution_directory)
     mic_dir = user_execution_directory / MIC_DIR
     create_base_directories(mic_dir)
+    print("making girignore")
+    render_gitignore(mic_dir)
+    print("girignore DONE")
     mic_config_path = create_config_file_yaml(mic_dir)
     framework = detect_framework_main(user_execution_directory)
-    image = build_docker(mic_dir / DOCKER_DIR, name)
-    if not image:
-        click.secho("The extraction of dependencies has failed", fg='red')
-        click.secho("Running a Docker Container without your dependencies. Please install them manually", fg='green')
-        image = framework.image
+    if image is None:
+        image = build_docker(mic_dir / DOCKER_DIR, name)
+        if not image:
+            click.secho("The extraction of dependencies has failed", fg='red')
+            click.secho("Running a Docker Container without your dependencies. Please install them manually", fg='green')
+            image = framework.image
     write_spec(mic_config_path, NAME_KEY, name)
     write_spec(mic_config_path, DOCKER_KEY, image)
     write_spec(mic_config_path, FRAMEWORK_KEY, framework)
@@ -158,7 +165,7 @@ def trace(command, c, o):
     default=None
 )
 @click.option('-a', '--auto_param', is_flag=True, default=False, help="Enable automatic detection of parameters")
-def configs(mic_file, configuration_files,auto_param):
+def configs(mic_file, configuration_files, auto_param):
     """
     Note: If your model does not use configuration files, you can skip this step
 
@@ -203,7 +210,7 @@ def configs(mic_file, configuration_files,auto_param):
 
 
 @cli.command(short_help="Expose parameters in the " + CONFIG_YAML_NAME + " file", name="parameters")
-@click.option('--name', "-n",  help="Name of the parameter", required=True, type=click.STRING)
+@click.option('--name', "-n", help="Name of the parameter", required=True, type=click.STRING)
 @click.option('--value', "-v", help="Default value of the parameter", required=True, type=ANY_TYPE)
 @click.option('--description', "-d", help="Description for parameter", required=False, type=str)
 @click.option('--overwrite', "-o", help="Overwrite an existing parameter", is_flag=True, default=False)
@@ -394,7 +401,8 @@ def outputs(mic_file, custom_outputs):
     write_spec(mic_config_file, OUTPUTS_KEY, relative(outputs, user_execution_directory))
 
 
-@cli.command(short_help=f"""Generate the directory structure and commands required to run your model component using the information from the
+@cli.command(
+    short_help=f"""Generate the directory structure and commands required to run your model component using the information from the
 previous steps""")
 @click.option(
     "-f",
@@ -486,7 +494,8 @@ def run(mic_file):
         info_end_run_failed()
 
 
-@cli.command(short_help="Publish your code on GitHub, your image on DockerHub and your model component on the MINT Model Catalog.")
+@cli.command(
+    short_help="Publish your code on GitHub, your image on DockerHub and your model component on the MINT Model Catalog.")
 @click.option(
     "-f",
     "--mic_file",
@@ -501,7 +510,9 @@ def run(mic_file):
     default="default",
     metavar="<profile-name>",
 )
-def publish(mic_file, profile):
+@click.option('--model_catalog_type',
+              type=click.Choice([i.label for i in ModelCatalogTypes], case_sensitive=False), default=ModelCatalogTypes.MODEL_CONFIGURATION.label)
+def publish(mic_file, profile, model_catalog_type):
     """
   Publish your MIC wrapper (including all the contents of the /src folder) on GitHub, the Docker Image on DockerHub
   and the model component on MINT Model Catalog.
@@ -516,13 +527,17 @@ def publish(mic_file, profile):
   mic encapsulate publish -f mic/mic.yaml
     """
     # Searches for mic file if user does not provide one
+    model_catalog_type = ModelCatalogTypes(model_catalog_type)
     mic_file = check_mic_path(mic_file)
-
     info_start_publish()
     mic_config_path = Path(mic_file)
     name = get_key_spec(mic_config_path, NAME_KEY)
     push(mic_config_path.parent, mic_config_path, name, profile)
     publish_docker(mic_config_path, name, profile)
-    model_configuration = create_model_catalog_resource(Path(mic_file), name, allow_local_path=False)
-    api_response_model, api_response_mc, model_id, software_version_id = publish_model_configuration(model_configuration, profile)
-    info_end_publish(obtain_id(model_id), obtain_id(software_version_id), obtain_id(api_response_mc.id), profile)
+    if model_catalog_type == ModelCatalogTypes.MODEL_CONFIGURATION:
+        model_configuration = create_model_catalog_resource(Path(mic_file), name, allow_local_path=False)
+        api_response_model, api_response_mc, model_id, software_version_id = publish_model_configuration(
+            model_configuration, profile)
+        info_end_publish(obtain_id(model_id), obtain_id(software_version_id), obtain_id(api_response_mc.id), profile)
+    elif model_catalog_type == ModelCatalogTypes.DATA_TRANSFORMATION:
+        click.secho("Add Data Transformation")

src/mic/component/initialization.py

@@ -43,12 +43,17 @@ def create_base_directories(mic_component_dir: Path, interactive=True):
 
 
 def render_gitignore(directory: Path):
+    """
+    creates a gitignore file (in given directory) from the gitignore template
+    :param directory:
+    :return:
+    """
     template = env.get_template(GITIGNORE_FILE)
     gitignore_file = directory / GITIGNORE_FILE
 
     with open(gitignore_file, "w") as gi:
         ignore = render_template(template=template)
-        gi.write(ignore)
+        gi.write("{}\n".format(ignore))
 
     gitignore_file.chmod(0o755)
 

src/mic/constants.py

@@ -69,6 +69,8 @@
 DEFAULT_CONFIGURATION_WARNING = "WARNING: The profile doesn't exists. To configure it, run:\nmic configure -p"
 DEFAULT_REGION = "texas"
 MODEL_CATALOG_URL = f"https://mint.isi.edu/{DEFAULT_REGION}/models/configure"
+
+
 class Framework(Enum):
     PYTHON37 = ("python3.7", "mintproject/python:3.7", ".py")
     PYTHON38 = ("python3.8", "mintproject/python:3.8", ".py")
@@ -85,6 +87,17 @@ def __str__(self):
         return self.label
 
 
+class ModelCatalogTypes(Enum):
+    MODEL_CONFIGURATION = ("Model Configuration")
+    DATA_TRANSFORMATION = ("Data Transformation")
+
+    def __init__(self, label):
+        self.label = label
+
+    def __str__(self):
+        return self.label
+
+
 def handle(value):
     for i in Framework:
         if value == i.label:

src/mic/publisher/github.py

@@ -124,7 +124,10 @@ def git_add_remote(repo, url):
 
 def git_pull(repo, remote, branch="master"):
     remote.fetch()
-    remote_master_id = repo.lookup_reference('refs/remotes/origin/%s' % (branch)).target
+    try:
+        remote_master_id = repo.lookup_reference('refs/remotes/origin/%s' % (branch)).target
+    except KeyError:
+        return
     merge_result, _ = repo.merge_analysis(remote_master_id)
     # Up to date, do nothing
     if merge_result & pygit2.GIT_MERGE_ANALYSIS_UP_TO_DATE:

src/mic/tests/resources/185/.reprozip-trace/config.yml

@@ -0,0 +1,170 @@
+# ReproZip configuration file
+version: '0.8'  # This file was generated by reprozip 1.0.16 at 2020-07-02T18:17:38-07:00
+runs:
+# You might want to edit this file before running the packer
+# See 'reprozip pack -h' for help
+- architecture: x86_64
+# Run info
+  argv:
+  - /bin/sh
+# Run 0
+  - ./addtoarray.sh
+  - '12.0'
+  binary: /bin/sh
+  distribution:
+  - ubuntu
+  - '20.04'
+  environ:
+    COLORTERM: truecolor
+    DISPLAY: :1
+    HOME: /root
+    LANG: en_US.UTF-8
+    LESSCLOSE: /usr/bin/lesspipe %s %s
+    LESSOPEN: '| /usr/bin/lesspipe %s'
+    LOGNAME: root
+    LS_COLORS: 'rs=0:di=01;34:ln=01;36:mh=00:pi=40;33:so=01;35:do=01;35:bd=40;33;01:cd=40;33;01:or=40;31;01:mi=00:su=37;41:sg=30;43:ca=30;41:tw=30;42:ow=34;42:st=37;44:ex=01;32:*.tar=01;31:*.tgz=01;31:*.arc=01;31:*.arj=01;31:*.taz=01;31:*.lha=01;31:*.lz4=01;31:*.lzh=01;31:*.lzma=01;31:*.tlz=01;31:*.txz=01;31:*.tzo=01;31:*.t7z=01;31:*.zip=01;31:*.z=01;31:*.dz=01;31:*.gz=01;31:*.lrz=01;31:*.lz=01;31:*.lzo=01;31:*.xz=01;31:*.zst=01;31:*.tzst=01;31:*.bz2=01;31:*.bz=01;31:*.tbz=01;31:*.tbz2=01;31:*.tz=01;31:*.deb=01;31:*.rpm=01;31:*.jar=01;31:*.war=01;31:*.ear=01;31:*.sar=01;31:*.rar=01;31:*.alz=01;31:*.ace=01;31:*.zoo=01;31:*.cpio=01;31:*.7z=01;31:*.rz=01;31:*.cab=01;31:*.wim=01;31:*.swm=01;31:*.dwm=01;31:*.esd=01;31:*.jpg=01;35:*.jpeg=01;35:*.mjpg=01;35:*.mjpeg=01;35:*.gif=01;35:*.bmp=01;35:*.pbm=01;35:*.pgm=01;35:*.ppm=01;35:*.tga=01;35:*.xbm=01;35:*.xpm=01;35:*.tif=01;35:*.tiff=01;35:*.png=01;35:*.svg=01;35:*.svgz=01;35:*.mng=01;35:*.pcx=01;35:*.mov=01;35:*.mpg=01;35:*.mpeg=01;35:*.m2v=01;35:*.mkv=01;35:*.webm=01;35:*.ogm=01;35:*.mp4=01;35:*.m4v=01;35:*.mp4v=01;35:*.vob=01;35:*.qt=01;35:*.nuv=01;35:*.wmv=01;35:*.asf=01;35:*.rm=01;35:*.rmvb=01;35:*.flc=01;35:*.avi=01;35:*.fli=01;35:*.flv=01;35:*.gl=01;35:*.dl=01;35:*.xcf=01;35:*.xwd=01;35:*.yuv=01;35:*.cgm=01;35:*.emf=01;35:*.ogv=01;35:*.ogx=01;35:*.aac=00;36:*.au=00;36:*.flac=00;36:*.m4a=00;36:*.mid=00;36:*.midi=00;36:*.mka=00;36:*.mp3=00;36:*.mpc=00;36:*.ogg=00;36:*.ra=00;36:*.wav=00;36:*.oga=00;36:*.opus=00;36:*.spx=00;36:*.xspf=00;36:'
+    MAIL: /var/mail/root
+    MODEL_CATALOG_PASSWORD: jz2KNTg5XgFacX4
+    OLDPWD: /tmp
+    PATH: /usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/usr/games:/usr/local/games
+    PWD: /tmp/mint/
+    SHELL: /bin/bash
+    SHLVL: '1'
+    SUDO_COMMAND: /usr/bin/su
+    SUDO_GID: '1000'
+    SUDO_UID: '1000'
+    SUDO_USER: chris
+    TERM: xterm-256color
+    USER: root
+    XAUTHORITY: /run/user/1000/gdm/Xauthority
+    _: /usr/local/bin/mic
+  exitcode: 0
+  gid: 0
+  hostname: chris-ubuntu20
+  id: run0
+  system:
+  - Linux
+  - 5.4.0-39-generic
+  uid: 0
+  workingdir: /tmp/mint
+inputs_outputs:
+- name: arg
+  path: /tmp/mint/addtoarray.sh
+  written_by_runs: []
+# Input and output files
+  read_by_runs:
+# Inputs are files that are only read by a run; reprounzip can replace these
+# files on demand to run the experiment with custom data.
+# Outputs are files that are generated by a run; reprounzip can extract these
+# files from the experiment on demand, for the user to examine.
+# The name field is the identifier the user will use to access these files.
+  - 0
+- name: in.txt
+  path: /tmp/mint/in.txt
+  written_by_runs: []
+  read_by_runs:
+  - 0
+- name: out.csv
+  path: /tmp/mint/outputs/out.csv
+  written_by_runs:
+  - 0
+  read_by_runs: []
+packages:
+- name: coreutils
+  version: 8.30-3ubuntu2
+# Files to pack
+# All the files below were used by the program; they will be included in the
+# generated package
+  size: 7368704
+# These files come from packages; we can thus choose not to include them, as it
+# will simply be possible to install that package on the destination system
+# They are included anyway by default
+  packfiles: true
+  files:
+  - /usr/bin/expr
+- name: language-pack-en-base
+  version: 1:20.04+20200416
+  size: 3900416
+  packfiles: true  # Total files used: 54.34 KB
+  files:  # Installed package size: 7.03 MB
+# 54.34 KB
+  - /usr/share/locale-langpack/en/LC_MESSAGES/coreutils.mo
+- name: libpcre2-8-0
+  version: 10.34-7
+  size: 606208
+  packfiles: true
+  files:  # Total files used: 613.0 bytes
+  - /usr/lib/x86_64-linux-gnu/libpcre2-8.so.0  # Installed package size: 3.72 MB
+# 613.0 bytes
+  - /usr/lib/x86_64-linux-gnu/libpcre2-8.so.0.9.0
+- name: locales
+  version: 2.31-0ubuntu9
+  size: 17608704
+  packfiles: true
+  files:  # Total files used: 570.70 KB
+  - /etc/locale.alias  # Installed package size: 592.00 KB
+# Link to /usr/lib/x86_64-linux-gnu/libpcre2-8.so.0.9.0
+# 570.70 KB
+  - /usr/share/locale/locale.alias
+other_files:
+- /bin
+- /etc/ld.so.cache
+- /tmp/mint
+- /tmp/mint/addtoarray.sh  # Total files used: 2.93 KB
+- /tmp/mint/in.txt  # Installed package size: 16.79 MB
+# 2.93 KB
+# Link to /etc/locale.alias
+- /lib
+# These files do not appear to come with an installed package -- you probably
+# want them packed
+- /lib64
+# Link to /usr/bin
+# 82.78 KB
+# Directory
+# 137.0 bytes
+# 26.0 bytes
+# Link to /usr/lib
+# Link to /usr/lib64
+# 42.40 KB
+# 126.77 KB
+# 86.34 KB
+# Link to /usr/bin/dash
+# 98.37 KB
+# 176.39 KB
+# Link to /usr/lib/i386-linux-gnu/ld-2.31.so
+# 13.86 MB
+# 186.98 KB
+# 1.94 MB
+# Link to /usr/lib/x86_64-linux-gnu/libc-2.31.so
+# 18.38 KB
+# Link to /usr/lib/x86_64-linux-gnu/libdl-2.31.so
+# 153.54 KB
+# Link to /usr/lib/x86_64-linux-gnu/libpthread-2.31.so
+# 159.38 KB
+# Link to /lib/x86_64-linux-gnu/ld-2.31.so
+- /usr/bin/cat
+# If you want to include additional files in the pack, you can list additional
+# patterns of files that will be included
+- /usr/bin/dash
+- /usr/bin/mkdir  # Example:
+#  - /etc/apache2/**  # Everything under apache2/
+#  - /var/log/apache2/*.log  # Log files directly under apache2/
+#  - /var/lib/lxc/*/rootfs/home/**/*.py  # All Python files of all users in
+#    # that container
+- /usr/bin/sh
+- /usr/bin/touch
+- /usr/lib/i386-linux-gnu/ld-2.31.so
+- /usr/lib/ld-linux.so.2
+- /usr/lib/locale/locale-archive
+- /usr/lib/x86_64-linux-gnu/ld-2.31.so
+- /usr/lib/x86_64-linux-gnu/libc-2.31.so
+- /usr/lib/x86_64-linux-gnu/libc.so.6
+- /usr/lib/x86_64-linux-gnu/libdl-2.31.so
+- /usr/lib/x86_64-linux-gnu/libdl.so.2
+- /usr/lib/x86_64-linux-gnu/libpthread-2.31.so
+- /usr/lib/x86_64-linux-gnu/libpthread.so.0
+- /usr/lib/x86_64-linux-gnu/libselinux.so.1
+- /usr/lib64/ld-linux-x86-64.so.2
+additional_patterns: null
+outputs:
+- /tmp/mint/outputs/out.txt

src/mic/tests/resources/185/addtoarray.sh

@@ -0,0 +1,7 @@
+mkdir outputs
+touch outputs/out.csv
+for i in $(cat in.txt); 
+do 
+	echo "writing: $1 + $i"
+	echo `expr $1 + $i` >> outputs/out.csv; 
+done

src/mic/tests/resources/185/in.txt

@@ -0,0 +1,12 @@
+1
+23
+43
+2
+34
+21
+23
+54
+3
+21
+2
+

src/mic/tests/resources/185/mic/docker/Dockerfile

@@ -0,0 +1,2 @@
+FROM mintproject/generic:latest
+