.gitlab-ci.yml

# Note: expand yaml
# yaml merge-expand .gitlab-ci.yml _expandyml && cat _expandyml
# 
# GITLAB LINTER
# https://gitlab.kitware.com/computer-vision/netharn/-/ci/lint


# This CI file has 4 types of jobs:
# (1) in the build stage we we build the wheels on a manylinux docker image
# (2) then in the test stage we install the wheels, run unit tests, and measure coverage
# (3) after testing we sign the wheels with the CI's GPG key
# (4) finally if we are on the release branch we will push the signed wheels to pypi

stages:
  - build
  - test
  - gpgsign
  - deploy


### TEMPLATES ###
# Define common templates using YAML anchors

.common_template: &common_template
    tags:
        # Tags define which runners will accept which jobs
        - docker
        - linux
        - build

    variables:
        # Change pip's cache directory to be inside the project directory since we can
        # only cache local items.
        PIP_CACHE_DIR: "$CI_PROJECT_DIR/mb_work/cache_pip"


.build_template: &build_template
    # Tags define which runners will accept which jobs
    <<: 
        - *common_template

    stage: 
        build
    
    before_script:
        - python -V  # Print out python version for debugging

    script: 
        - python setup.py bdist_wheel --universal

    cache:
        paths:
            - .cache/pip

    artifacts:
        paths:
            - dist/*.whl


.test_full_template: &test_full_template
    # Tags define which runners will accept which jobs
    <<: 
        - *common_template

    stage: 
        test
    
    before_script:
        - python -V  # Print out python version for debugging
        - export PYVER=$(python -c "import sys; print('{}{}'.format(*sys.version_info[0:2]))")
        - pip install --progress-bar off virtualenv
        - virtualenv venv$PYVER
        - source venv$PYVER/bin/activate
        - pip install --progress-bar off pip -U
        - pip install --progress-bar off pip setuptools -U
        - python -V  # Print out python version for debugging
        - pip install --progress-bar off -r requirements.txt
        - pip install .
        # FIXME: we should start from a docker iamge that already has LibGL setup
        - apt update && apt install libgl1-mesa-glx -y && rm -rf /var/lib/apt/lists/*  

    script: 
        - ./run_tests.py

    cache:
        paths:
            - .cache/pip
            - venv/

    # Coverage is a regex that will parse the coverage from the test stdout
    coverage: '/TOTAL.+ ([0-9]{1,3}%)/'


.__gpg_heredoc__: &__gpg_heredoc__
        - | 
            # THIS IS NOT EXECUTE ON THE CI, THIS IS FOR DEVELOPER REFERENCE
            # ON HOW THE ENCRYPTED GPG KEYS ARE SETUP.
            
            # Load or generate secrets
            source $(secret_loader.sh)
            echo $CI_KITWARE_SECRET
            echo $TWINE_USERNAME

            # ADD RELEVANT VARIABLES TO GITLAB SECRET VARIABLES
            # https://gitlab.kitware.com/computer-vision/netharn/-/settings/ci_cd
            # Note that it is important to make sure that these variables are
            # only decrpyted on protected branches by selecting the protected
            # and masked option. Also make sure you have master and release
            # branches protected.
            # https://gitlab.kitware.com/computer-vision/netharn/-/settings/repository#js-protected-branches-settings

            # HOW TO ENCRYPT YOUR SECRET GPG KEY
            IDENTIFIER="travis-ci-Erotemic"
            GPG_KEYID=$(gpg --list-keys --keyid-format LONG "$IDENTIFIER" | head -n 2 | tail -n 1 | awk '{print $1}' | tail -c 9)
            echo "GPG_KEYID = $GPG_KEYID"

            # Export plaintext gpg public keys, private keys, and trust info
            mkdir -p dev
            gpg --armor --export-secret-keys $GPG_KEYID > dev/ci_secret_gpg_key.pgp
            gpg --armor --export $GPG_KEYID > dev/ci_public_gpg_key.pgp
            gpg --export-ownertrust > dev/gpg_owner_trust

            # Encrypt gpg keys and trust with CI secret
            GLKWS=$CI_KITWARE_SECRET openssl enc -aes-256-cbc -pbkdf2 -md SHA512 -pass env:GLKWS -e -a -in dev/ci_public_gpg_key.pgp > dev/ci_public_gpg_key.pgp.enc
            GLKWS=$CI_KITWARE_SECRET openssl enc -aes-256-cbc -pbkdf2 -md SHA512 -pass env:GLKWS -e -a -in dev/ci_secret_gpg_key.pgp > dev/ci_secret_gpg_key.pgp.enc
            GLKWS=$CI_KITWARE_SECRET openssl enc -aes-256-cbc -pbkdf2 -md SHA512 -pass env:GLKWS -e -a -in dev/gpg_owner_trust > dev/gpg_owner_trust.enc
            echo $GPG_KEYID > dev/public_gpg_key

            # Test decrpyt
            cat dev/public_gpg_key
            GLKWS=$CI_KITWARE_SECRET openssl enc -aes-256-cbc -pbkdf2 -md SHA512 -pass env:GLKWS -d -a -in dev/ci_public_gpg_key.pgp.enc 
            GLKWS=$CI_KITWARE_SECRET openssl enc -aes-256-cbc -pbkdf2 -md SHA512 -pass env:GLKWS -d -a -in dev/gpg_owner_trust.enc 
            GLKWS=$CI_KITWARE_SECRET openssl enc -aes-256-cbc -pbkdf2 -md SHA512 -pass env:GLKWS -d -a -in dev/ci_secret_gpg_key.pgp.enc 

            source $(secret_unloader.sh)

            # Look at what we did, clean up, and add it to git
            ls dev/*.enc
            rm dev/gpg_owner_trust dev/*.pgp
            git status
            git add dev/*.enc
            git add dev/public_gpg_key


.gpgsign_template: &gpgsign_template
    <<: 
        - *common_template

    stage: 
        gpgsign

    script: 
        - export GPG_EXECUTABLE=gpg
        - export GPG_KEYID=$(cat dev/public_gpg_key)
        - echo "GPG_KEYID = $GPG_KEYID"
        - $GPG_EXECUTABLE --version
        - openssl version
        - $GPG_EXECUTABLE --list-keys
        - $GPG_EXECUTABLE --list-keys
        # Decrypt and import GPG Keys / trust
        # note CI_KITWARE_SECRET is a protected variables only available on master and release branch
        - GLKWS=$CI_KITWARE_SECRET openssl enc -aes-256-cbc -pbkdf2 -md SHA512 -pass env:GLKWS -d -a -in dev/ci_public_gpg_key.pgp.enc | $GPG_EXECUTABLE --import 
        - GLKWS=$CI_KITWARE_SECRET openssl enc -aes-256-cbc -pbkdf2 -md SHA512 -pass env:GLKWS -d -a -in dev/gpg_owner_trust.enc | $GPG_EXECUTABLE --import-ownertrust
        - GLKWS=$CI_KITWARE_SECRET openssl enc -aes-256-cbc -pbkdf2 -md SHA512 -pass env:GLKWS -d -a -in dev/ci_secret_gpg_key.pgp.enc | $GPG_EXECUTABLE --import 
        - $GPG_EXECUTABLE --list-keys  || echo "first one fails for some reason"
        - $GPG_EXECUTABLE --list-keys  
        # The publish script only builds wheels and does gpg signing if TAG_AND_UPLOAD is False
        - MB_PYTHON_TAG=$MB_PYTHON_TAG USE_GPG=True GPG_KEYID=$GPG_KEYID TWINE_PASSWORD=$TWINE_PASSWORD TWINE_USERNAME=$TWINE_USERNAME GPG_EXECUTABLE=$GPG_EXECUTABLE DEPLOY_BRANCH=release TAG_AND_UPLOAD=False ./publish.sh 

    artifacts:
        paths:
            - dist/*.asc
            - dist/*.tar.gz
            - dist/*.whl

    only:
        refs:
            # Gitlab will only expose protected variables on protected branches
            # (which I've set to be master and release), so only run this stage
            # there.
            - master
            - release


.deploy_template: &deploy_template
    <<: 
        - *common_template

    stage: 
        deploy

    script: 
        - export GPG_EXECUTABLE=gpg
        - export GPG_KEYID=$(cat dev/public_gpg_key)
        - echo "GPG_KEYID = $GPG_KEYID"
        - $GPG_EXECUTABLE --version
        - openssl version
        - $GPG_EXECUTABLE --list-keys
        - $GPG_EXECUTABLE --list-keys
        # Decrypt and import GPG Keys / trust
        # note CI_KITWARE_SECRET is a protected variables only available on master and release branch
        - GLKWS=$CI_KITWARE_SECRET openssl enc -aes-256-cbc -pbkdf2 -md SHA512 -pass env:GLKWS -d -a -in dev/ci_public_gpg_key.pgp.enc | $GPG_EXECUTABLE --import 
        - GLKWS=$CI_KITWARE_SECRET openssl enc -aes-256-cbc -pbkdf2 -md SHA512 -pass env:GLKWS -d -a -in dev/gpg_owner_trust.enc | $GPG_EXECUTABLE --import-ownertrust
        - GLKWS=$CI_KITWARE_SECRET openssl enc -aes-256-cbc -pbkdf2 -md SHA512 -pass env:GLKWS -d -a -in dev/ci_secret_gpg_key.pgp.enc | $GPG_EXECUTABLE --import 
        - $GPG_EXECUTABLE --list-keys  || echo "first one fails for some reason"
        - $GPG_EXECUTABLE --list-keys  
        # Install twine
        - pip install six pyopenssl ndg-httpsclient pyasn1 -U 
        - pip install requests[security] twine 
        # Execute the publish script for real this time
        - MB_PYTHON_TAG=$MB_PYTHON_TAG USE_GPG=True GPG_KEYID=$GPG_KEYID TWINE_PASSWORD=$TWINE_PASSWORD TWINE_USERNAME=$TWINE_USERNAME GPG_EXECUTABLE=$GPG_EXECUTABLE CURRENT_BRANCH=release DEPLOY_BRANCH=release TAG_AND_UPLOAD=yes ./publish.sh 
        # Have the server git-tag the release and push the tags
        - VERSION=$(python -c "import setup; print(setup.VERSION)")
        # do sed twice to handle the case of https clone with and without a read token
        - URL_HOST=$(git remote get-url origin | sed -e 's|https\?://.*@||g' | sed -e 's|https\?://||g')
        - echo "URL_HOST = $URL_HOST"
        - git config user.email "ci@gitlab.kitware.com"
        - git config user.name "Gitlab-CI"
        - git tag $VERSION -m "tarball tag $VERSION"
        - git push --tags "https://${GIT_PUSH_TOKEN}@${URL_HOST}"
            
    only:
        refs:
            - release

.build_install_test: &build_install_test
    - pip install -r requirements.txt -U


### JOBS ###
# Define the actual jobs

# ---------------
# Python 3.8 Jobs

build/cp38-cp38-linux:
    <<: 
        - *build_template
    image:
        python:3.8

test_full/cp38-cp38-linux:
    <<: 
        - *test_full_template
    image:
        python:3.8


# for universal builds we only need to gpg sign once
gpgsign/cp38-cp38-linux:
    <<: 
        - *gpgsign_template
    image:
        python:3.8

deploy/cp38-cp38-linux:
    <<: 
        - *deploy_template
    image:
        python:3.8


# ---------------
# Python 3.7 Jobs

build/cp37-cp37m-linux:
    <<: 
        - *build_template
    image:
        python:3.7

test_full/cp37-cp37m-linux:
    <<: 
        - *test_full_template
    image:
        python:3.7


# ---------------
# Python 3.6 Jobs

build/cp36-cp36m-linux:
    <<: 
        - *build_template
    image:
        python:3.6

test_full/cp36-cp36m-linux:
    <<: 
        - *test_full_template
    image:
        python:3.6

#gpgsign/cp36-cp36m-linux:
#    <<: 
#        - *gpgsign_template
#    image:
#        python:3.6

#deploy/cp36-cp36m-linux:
#    <<: 
#        - *deploy_template
#    image:
#        python:3.6


# ---------------
# Python 3.5 Jobs

build/cp35-cp35m-linux:
    <<: 
        - *build_template
    image:
        python:3.5

test_full/cp35-cp35m-linux:
    <<: 
        - *test_full_template
    image:
        python:3.5

#gpgsign/cp35-cp35m-linux:
#    <<: 
#        - *gpgsign_template
#    image:
#        python:3.5

#deploy/cp35-cp35m-linux:
#    <<: 
#        - *deploy_template
#    image:
#        python:3.5


# ---------------
# Python 2.7 Jobs

#build/cp27-cp27mu-linux:
#    <<: 
#        - *build_template
#    image:
#        python:2.7

#test_full/cp27-cp27mu-linux:
#    <<: 
#        - *test_full_template
#    image:
#        python:2.7

#gpgsign/cp27-cp27mu-linux:
#    <<: 
#        - *gpgsign_template
#    image:
#        python:2.7

#deploy/cp27-cp27mu-linux:
#    <<: 
#        - *deploy_template
#    image:
#        python:2.7