Merge branch 'develop' of github.com:mintproject/mic into develop

src/mic/click_encapsulate/commands.py

@@ -13,7 +13,7 @@
 from mic.component.detect import detect_framework_main, detect_new_reprozip, extract_dependencies
 from mic.component.executor import copy_code_to_src, compress_directory, execute_local, copy_config_to_src
 from mic.component.initialization import render_run_sh, render_io_sh, render_output, create_base_directories, \
-    render_bash_color
+    render_bash_color, render_gitignore
 from mic.component.reprozip import get_inputs_outputs_reprozip, get_outputs_reprozip, relative, generate_runner, \
     generate_pre_runner, \
     find_code_files
@@ -70,6 +70,9 @@ def start(user_execution_directory, name, image):
     user_execution_directory = Path(user_execution_directory)
     mic_dir = user_execution_directory / MIC_DIR
     create_base_directories(mic_dir)
+    print("making girignore")
+    render_gitignore(mic_dir)
+    print("girignore DONE")
     mic_config_path = create_config_file_yaml(mic_dir)
     framework = detect_framework_main(user_execution_directory)
     if image is None:

src/mic/component/initialization.py

@@ -43,12 +43,17 @@ def create_base_directories(mic_component_dir: Path, interactive=True):
 
 
 def render_gitignore(directory: Path):
+    """
+    creates a gitignore file (in given directory) from the gitignore template
+    :param directory:
+    :return:
+    """
     template = env.get_template(GITIGNORE_FILE)
     gitignore_file = directory / GITIGNORE_FILE
 
     with open(gitignore_file, "w") as gi:
         ignore = render_template(template=template)
-        gi.write(ignore)
+        gi.write("{}\n".format(ignore))
 
     gitignore_file.chmod(0o755)
 

src/mic/tests/resources/185/.reprozip-trace/config.yml

@@ -0,0 +1,170 @@
+# ReproZip configuration file
+version: '0.8'  # This file was generated by reprozip 1.0.16 at 2020-07-02T18:17:38-07:00
+runs:
+# You might want to edit this file before running the packer
+# See 'reprozip pack -h' for help
+- architecture: x86_64
+# Run info
+  argv:
+  - /bin/sh
+# Run 0
+  - ./addtoarray.sh
+  - '12.0'
+  binary: /bin/sh
+  distribution:
+  - ubuntu
+  - '20.04'
+  environ:
+    COLORTERM: truecolor
+    DISPLAY: :1
+    HOME: /root
+    LANG: en_US.UTF-8
+    LESSCLOSE: /usr/bin/lesspipe %s %s
+    LESSOPEN: '| /usr/bin/lesspipe %s'
+    LOGNAME: root
+    LS_COLORS: 'rs=0:di=01;34:ln=01;36:mh=00:pi=40;33:so=01;35:do=01;35:bd=40;33;01:cd=40;33;01:or=40;31;01:mi=00:su=37;41:sg=30;43:ca=30;41:tw=30;42:ow=34;42:st=37;44:ex=01;32:*.tar=01;31:*.tgz=01;31:*.arc=01;31:*.arj=01;31:*.taz=01;31:*.lha=01;31:*.lz4=01;31:*.lzh=01;31:*.lzma=01;31:*.tlz=01;31:*.txz=01;31:*.tzo=01;31:*.t7z=01;31:*.zip=01;31:*.z=01;31:*.dz=01;31:*.gz=01;31:*.lrz=01;31:*.lz=01;31:*.lzo=01;31:*.xz=01;31:*.zst=01;31:*.tzst=01;31:*.bz2=01;31:*.bz=01;31:*.tbz=01;31:*.tbz2=01;31:*.tz=01;31:*.deb=01;31:*.rpm=01;31:*.jar=01;31:*.war=01;31:*.ear=01;31:*.sar=01;31:*.rar=01;31:*.alz=01;31:*.ace=01;31:*.zoo=01;31:*.cpio=01;31:*.7z=01;31:*.rz=01;31:*.cab=01;31:*.wim=01;31:*.swm=01;31:*.dwm=01;31:*.esd=01;31:*.jpg=01;35:*.jpeg=01;35:*.mjpg=01;35:*.mjpeg=01;35:*.gif=01;35:*.bmp=01;35:*.pbm=01;35:*.pgm=01;35:*.ppm=01;35:*.tga=01;35:*.xbm=01;35:*.xpm=01;35:*.tif=01;35:*.tiff=01;35:*.png=01;35:*.svg=01;35:*.svgz=01;35:*.mng=01;35:*.pcx=01;35:*.mov=01;35:*.mpg=01;35:*.mpeg=01;35:*.m2v=01;35:*.mkv=01;35:*.webm=01;35:*.ogm=01;35:*.mp4=01;35:*.m4v=01;35:*.mp4v=01;35:*.vob=01;35:*.qt=01;35:*.nuv=01;35:*.wmv=01;35:*.asf=01;35:*.rm=01;35:*.rmvb=01;35:*.flc=01;35:*.avi=01;35:*.fli=01;35:*.flv=01;35:*.gl=01;35:*.dl=01;35:*.xcf=01;35:*.xwd=01;35:*.yuv=01;35:*.cgm=01;35:*.emf=01;35:*.ogv=01;35:*.ogx=01;35:*.aac=00;36:*.au=00;36:*.flac=00;36:*.m4a=00;36:*.mid=00;36:*.midi=00;36:*.mka=00;36:*.mp3=00;36:*.mpc=00;36:*.ogg=00;36:*.ra=00;36:*.wav=00;36:*.oga=00;36:*.opus=00;36:*.spx=00;36:*.xspf=00;36:'
+    MAIL: /var/mail/root
+    MODEL_CATALOG_PASSWORD: jz2KNTg5XgFacX4
+    OLDPWD: /tmp
+    PATH: /usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/usr/games:/usr/local/games
+    PWD: /tmp/mint/
+    SHELL: /bin/bash
+    SHLVL: '1'
+    SUDO_COMMAND: /usr/bin/su
+    SUDO_GID: '1000'
+    SUDO_UID: '1000'
+    SUDO_USER: chris
+    TERM: xterm-256color
+    USER: root
+    XAUTHORITY: /run/user/1000/gdm/Xauthority
+    _: /usr/local/bin/mic
+  exitcode: 0
+  gid: 0
+  hostname: chris-ubuntu20
+  id: run0
+  system:
+  - Linux
+  - 5.4.0-39-generic
+  uid: 0
+  workingdir: /tmp/mint
+inputs_outputs:
+- name: arg
+  path: /tmp/mint/addtoarray.sh
+  written_by_runs: []
+# Input and output files
+  read_by_runs:
+# Inputs are files that are only read by a run; reprounzip can replace these
+# files on demand to run the experiment with custom data.
+# Outputs are files that are generated by a run; reprounzip can extract these
+# files from the experiment on demand, for the user to examine.
+# The name field is the identifier the user will use to access these files.
+  - 0
+- name: in.txt
+  path: /tmp/mint/in.txt
+  written_by_runs: []
+  read_by_runs:
+  - 0
+- name: out.csv
+  path: /tmp/mint/outputs/out.csv
+  written_by_runs:
+  - 0
+  read_by_runs: []
+packages:
+- name: coreutils
+  version: 8.30-3ubuntu2
+# Files to pack
+# All the files below were used by the program; they will be included in the
+# generated package
+  size: 7368704
+# These files come from packages; we can thus choose not to include them, as it
+# will simply be possible to install that package on the destination system
+# They are included anyway by default
+  packfiles: true
+  files:
+  - /usr/bin/expr
+- name: language-pack-en-base
+  version: 1:20.04+20200416
+  size: 3900416
+  packfiles: true  # Total files used: 54.34 KB
+  files:  # Installed package size: 7.03 MB
+# 54.34 KB
+  - /usr/share/locale-langpack/en/LC_MESSAGES/coreutils.mo
+- name: libpcre2-8-0
+  version: 10.34-7
+  size: 606208
+  packfiles: true
+  files:  # Total files used: 613.0 bytes
+  - /usr/lib/x86_64-linux-gnu/libpcre2-8.so.0  # Installed package size: 3.72 MB
+# 613.0 bytes
+  - /usr/lib/x86_64-linux-gnu/libpcre2-8.so.0.9.0
+- name: locales
+  version: 2.31-0ubuntu9
+  size: 17608704
+  packfiles: true
+  files:  # Total files used: 570.70 KB
+  - /etc/locale.alias  # Installed package size: 592.00 KB
+# Link to /usr/lib/x86_64-linux-gnu/libpcre2-8.so.0.9.0
+# 570.70 KB
+  - /usr/share/locale/locale.alias
+other_files:
+- /bin
+- /etc/ld.so.cache
+- /tmp/mint
+- /tmp/mint/addtoarray.sh  # Total files used: 2.93 KB
+- /tmp/mint/in.txt  # Installed package size: 16.79 MB
+# 2.93 KB
+# Link to /etc/locale.alias
+- /lib
+# These files do not appear to come with an installed package -- you probably
+# want them packed
+- /lib64
+# Link to /usr/bin
+# 82.78 KB
+# Directory
+# 137.0 bytes
+# 26.0 bytes
+# Link to /usr/lib
+# Link to /usr/lib64
+# 42.40 KB
+# 126.77 KB
+# 86.34 KB
+# Link to /usr/bin/dash
+# 98.37 KB
+# 176.39 KB
+# Link to /usr/lib/i386-linux-gnu/ld-2.31.so
+# 13.86 MB
+# 186.98 KB
+# 1.94 MB
+# Link to /usr/lib/x86_64-linux-gnu/libc-2.31.so
+# 18.38 KB
+# Link to /usr/lib/x86_64-linux-gnu/libdl-2.31.so
+# 153.54 KB
+# Link to /usr/lib/x86_64-linux-gnu/libpthread-2.31.so
+# 159.38 KB
+# Link to /lib/x86_64-linux-gnu/ld-2.31.so
+- /usr/bin/cat
+# If you want to include additional files in the pack, you can list additional
+# patterns of files that will be included
+- /usr/bin/dash
+- /usr/bin/mkdir  # Example:
+#  - /etc/apache2/**  # Everything under apache2/
+#  - /var/log/apache2/*.log  # Log files directly under apache2/
+#  - /var/lib/lxc/*/rootfs/home/**/*.py  # All Python files of all users in
+#    # that container
+- /usr/bin/sh
+- /usr/bin/touch
+- /usr/lib/i386-linux-gnu/ld-2.31.so
+- /usr/lib/ld-linux.so.2
+- /usr/lib/locale/locale-archive
+- /usr/lib/x86_64-linux-gnu/ld-2.31.so
+- /usr/lib/x86_64-linux-gnu/libc-2.31.so
+- /usr/lib/x86_64-linux-gnu/libc.so.6
+- /usr/lib/x86_64-linux-gnu/libdl-2.31.so
+- /usr/lib/x86_64-linux-gnu/libdl.so.2
+- /usr/lib/x86_64-linux-gnu/libpthread-2.31.so
+- /usr/lib/x86_64-linux-gnu/libpthread.so.0
+- /usr/lib/x86_64-linux-gnu/libselinux.so.1
+- /usr/lib64/ld-linux-x86-64.so.2
+additional_patterns: null
+outputs:
+- /tmp/mint/outputs/out.txt

src/mic/tests/resources/185/addtoarray.sh

@@ -0,0 +1,7 @@
+mkdir outputs
+touch outputs/out.csv
+for i in $(cat in.txt); 
+do 
+	echo "writing: $1 + $i"
+	echo `expr $1 + $i` >> outputs/out.csv; 
+done

src/mic/tests/resources/185/in.txt

@@ -0,0 +1,12 @@
+1
+23
+43
+2
+34
+21
+23
+54
+3
+21
+2
+

src/mic/tests/resources/185/mic/docker/Dockerfile

@@ -0,0 +1,2 @@
+FROM mintproject/generic:latest
+

src/mic/tests/resources/185/mic/mic.yaml

@@ -0,0 +1,7 @@
+step: 1
+name: '185'
+docker_image: 185:latest
+framework: !!python/object/apply:mic.constants.Framework
+- !!python/tuple
+  - general
+  - mintproject/generic:latest

src/mic/tests/resources/185/outputs/out.csv

@@ -0,0 +1,10 @@
+1
+21
+42
+6
+32
+46
+8
+7
+97
+2

src/mic/tests/resources/209/temp.txt

@@ -0,0 +1 @@
+github needs a folder to have a file in it. So here it is

src/mic/tests/test_185.py

@@ -0,0 +1,96 @@
+import shutil
+from os import fdopen
+from pathlib import Path
+from tempfile import mkstemp
+
+from click.testing import CliRunner
+from mic.config_yaml import get_parameters, get_inputs, get_configs, get_outputs_mic
+from mic.click_encapsulate.commands import inputs, add_parameters, configs, outputs, wrapper, run
+from mic.constants import MIC_DIR, CONFIG_YAML_NAME
+
+RESOURCES = "resources"
+mic_1 = Path(__file__).parent / RESOURCES / "mic_full.yaml"
+mic_empty = Path(__file__).parent / RESOURCES / "mic_empty.yaml"
+
+
+def test_issue_185(tmp_path):
+    """
+    Test case for:
+        1 input
+        1 output
+
+    :param tmp_path:
+    :return:
+    """
+
+    test_name = "185"
+
+    temp_test = tmp_path / test_name
+    mic_dir = temp_test / MIC_DIR
+    mic_config_arg = str(mic_dir / CONFIG_YAML_NAME)
+
+    path_test_name = tmp_path / test_name
+    path = Path(__file__).parent / RESOURCES / test_name
+    shutil.copytree(path, path_test_name)
+    runner = CliRunner()
+    cmd_add_parameters(mic_config_arg, runner)
+    check_parameters(mic_config_arg)
+    cmd_inputs(mic_config_arg, runner)
+    check_inputs(mic_config_arg)
+    cmd_outputs(mic_config_arg, runner)
+    check_outputs(mic_config_arg)
+    cmd_wrapper(mic_config_arg, runner)
+    cmd_run(mic_config_arg, runner)
+
+
+def check_parameters(mic_config_arg):
+    parameters = get_parameters(Path(mic_config_arg))
+    assert parameters == {'add': {'default_value': 12.0, 'description': '', 'type': 'float'}}
+
+
+def cmd_add_parameters(mic_config_arg, runner):
+    parameters = {"add": 12}
+    for key, value in parameters.items():
+
+        result = runner.invoke(add_parameters, ["-f", mic_config_arg, "--name", key, "--value", value],
+                               catch_exceptions=False)
+        print(result.output)
+        assert result.exit_code == 0
+
+
+def cmd_inputs(mic_config_arg, runner):
+
+    result = runner.invoke(inputs, ["-f", mic_config_arg], input='Y', catch_exceptions=False)
+    print(result.output)
+    assert result.exit_code == 0
+
+
+def check_inputs(mic_config_arg):
+    _inputs = get_inputs(Path(mic_config_arg))
+    assert _inputs == {'in_txt': {'format': 'txt', 'path': 'in.txt'},
+                       'outputs_zip': {'format': 'zip', 'path': 'outputs.zip'}}
+
+
+def cmd_outputs(mic_config_arg, runner):
+    result = runner.invoke(outputs, ["-f", mic_config_arg], catch_exceptions=False)
+    print(result.output)
+    assert result.exit_code == 0
+
+
+def check_outputs(mic_config_arg):
+    files = get_outputs_mic(Path(mic_config_arg))
+    assert files == {'out_txt': {'format': 'txt', 'path': 'outputs/out.txt'}}
+
+
+def cmd_wrapper(mic_config_arg, runner):
+    result = runner.invoke(wrapper, ["-f", mic_config_arg], catch_exceptions=False)
+    print(result.output)
+    assert result.exit_code == 0
+
+
+def cmd_run(mic_config_arg, runner):
+
+    result = runner.invoke(run, ["-f", mic_config_arg], catch_exceptions=False)
+    print(result.output)
+    assert result.exit_code == 0
+