From 74ae0fe8f14fc63b64997be0070067654ff3094a Mon Sep 17 00:00:00 2001
From: Kaushik Varanasi <kaushik.varanasi1@gmail.com>
Date: Sun, 19 Jul 2020 20:00:40 +0530
Subject: [PATCH] [MRG] Using Mamba in Pipelines (#1210)

* testing mamba installation. Simply install mamba from conda and try to create env

* added mamba after conda installation

* check if mamba installation succesful

* installing mamba before activating

* fix syntax errors

* Pin mamba to 0.4.0 and make pipelines exit on error with any bash command

* move tardis_env.sh into pipelines file

* fetch reference data fetching from bash scripts to azure pipelines file

* fix syntax error

* Added packet data

* Adding variable shellopts to check

* added shellopts=errexit so that build fails on non-zero error code

* testing if pipeline fails

* reverting changes, testing succesful

* remove unused lines

* testing, this should fail the pipeline

* Made all single lines where necessary
Added a step to fail build on error as suggested by epassaro
improved documentation to steps

* added back the git lfs pull code

* removed git lfs codes

* added git lfs to pipelines to make it faster
---
 azure-pipelines/simple_test_framework.yml | 57 ++++++++++++++---------
 ci-helpers/install_miniconda.sh           |  2 +
 ci-helpers/install_tardis_env.sh          | 15 +++++-
 3 files changed, 50 insertions(+), 24 deletions(-)

diff --git a/azure-pipelines/simple_test_framework.yml b/azure-pipelines/simple_test_framework.yml
index 71a681935e1..7cd26309978 100644
--- a/azure-pipelines/simple_test_framework.yml
+++ b/azure-pipelines/simple_test_framework.yml
@@ -22,41 +22,52 @@ jobs:
       matrix:
         linux:
           vm_Image: "Ubuntu-16.04"
-          conda: "/usr/share/miniconda"
         mac:
           vm_Image: "macOS-10.14"
-          miniconda.url: "http://repo.continuum.io/miniconda/Miniconda2-latest-mac-x86_64.sh"
-      maxParallel: 4
+      maxParallel: 2
 
     steps:
-      - bash: |
-          echo "##vso[task.prependpath]$CONDA/bin"
-        displayName: Add conda to PATH
+      - bash: echo "##vso[task.setvariable variable=shellopts]errexit"
+        displayName: "Force exit on error (bash) "
 
-      - bash: |
-          sudo chown -R $USER $CONDA
-          # conda update -y conda
-        displayName: updating conda and activating
+      - bash: echo "##vso[task.prependpath]$CONDA/bin"
+        displayName: "Add conda to PATH"
 
-      - bash: |
-          sh ci-helpers/install_tardis_env.sh
-        displayName: "Install TARDIS env"
+      - bash: sudo chown -R $USER $CONDA
+        condition: eq( variables['Agent.OS'], 'Darwin' )
+        displayName: "Take ownership of conda installation (macOS)"
+
+      - bash: conda install mamba=0.4.0 -c conda-forge -y
+        displayName: "Install mamba"
+
+      - bash: mamba env create -f tardis_env3.yml
+        displayName: "Install TARDIS environment"
 
       - bash: |
-          sh ci-helpers/fetch_reference_data.sh
-        displayName: "Fetch Reference Data"
+              echo git clone $(ref.data.github.url) $(ref.data.home)
+              git clone $(ref.data.github.url) $(ref.data.home)
+              cd $(ref.data.home)
+              git fetch origin
+              git checkout origin/master
+              git lfs pull --include="atom_data/kurucz_cd23_chianti_H_He.h5" origin
+              git lfs pull --include="atom_data/chianti_He.h5" origin
+              git lfs pull --include="plasma_reference/" origin
+              git lfs pull --include="unit_test_data.h5" origin
+              git lfs pull --include="packet_unittest.h5" origin
+              echo MD5: `md5sum unit_test_data.h5`
+        displayName: "Fetch reference data"
 
       - bash: |
-          source activate tardis
-          python setup.py build_ext --inplace
+              source activate tardis
+              python setup.py build_ext --inplace
         displayName: "TARDIS build"
 
       - bash: |
-          source activate tardis
-          pip install pytest-azurepipelines
-          pytest tardis --tardis-refdata=$(ref.data.home) --cov=tardis --cov-report=xml --cov-report=html
-
+              source activate tardis
+              pip install pytest-azurepipelines
+              pytest tardis --tardis-refdata=$(ref.data.home) --cov=tardis --cov-report=xml --cov-report=html
         displayName: "TARDIS test"
-      - script: |
-          bash <(curl -s https://codecov.io/bash)
+
+      - bash: |
+              bash <(curl -s https://codecov.io/bash)
         displayName: "Upload to codecov.io"
diff --git a/ci-helpers/install_miniconda.sh b/ci-helpers/install_miniconda.sh
index 2048d3b845c..5a3aafae5b7 100644
--- a/ci-helpers/install_miniconda.sh
+++ b/ci-helpers/install_miniconda.sh
@@ -6,6 +6,7 @@ if test -e $HOME/miniconda/bin; then
     hash -r
     #conda update --yes conda
 
+
 else
     wget $MINICONDA_URL -O miniconda.sh
     chmod +x miniconda.sh
@@ -15,6 +16,7 @@ else
     export PATH=$HOME/miniconda/bin:$PATH
     hash -r
     conda update --yes conda
+
 fi
 
 source $HOME/miniconda/etc/profile.d/conda.sh
\ No newline at end of file
diff --git a/ci-helpers/install_tardis_env.sh b/ci-helpers/install_tardis_env.sh
index c125887abf0..611bee21adc 100644
--- a/ci-helpers/install_tardis_env.sh
+++ b/ci-helpers/install_tardis_env.sh
@@ -1,10 +1,23 @@
 #!/usr/bin/env bash
 cd $TARDIS_BUILD_DIR
+
+# install mamba
+if test -e $HOME/miniconda/bin/mamba; then
+    echo "Mamba installed Succesfuly"
+
+else
+    echo "Mamba not installed"
+    conda install mamba=0.4.0 -c conda-forge -y
+    echo "Installed Mamba to correct location"
+fi
+
 if test -e  $HOME/miniconda/envs/tardis; then
     echo "TARDIS env already installed.";
     # Also check for tardis_env3.yml change
 else
-   conda env create -f tardis_env3.yml
+   # conda env create -f tardis_env3.yml
+   echo "Creating TARDIS environment using Mamba"
+   mamba env create -f tardis_env3.yml
    #trouble with building due to segfault at cython (https://github.com/cython/cython/issues/2199)
    #remove if we can get normal cython through conda
 fi