.gitlab-ci.yml

# for compatibility with old git versions on centos
variables:
  GIT_STRATEGY: clone
  K4_JOBTYPE:
    value: CI
    description: "Possible Values: Release, Nighlies, CI. A Release job is deployed to sw.hsf.org and uses the latest tags. A Nightlies job is deployed to sw-nightlies.hsf.org and uses the HEAD of the defined repos. A CI build is like a nightly build, but not deployed, and builds local checkouts of the PRs given in K4_CI_URLS"
  K4_CI_URLS:
    value: ""
    description: "A whitespace-separated list of urls for github prs (for repositories for which there is a spack recipe). Example: https://github.com/iLCSoft/SIO/pull/17/ https://github.com/AIDASoft/podio/pull/287"
  SPACK_VERSION:
    value: ""
    description: "A git ref to check out after cloning the spack repository. Leaving this blank will use the latest HEAD (recommended)."
  SPACK_INSTALL_ARGUMENTS:
    value: "--test=root --fresh --no-check-signature --show-log-on-error"
    description: "Arguments to be passed to spack install"

stages:
    - prepare-spack
    - compilation
    - buildtest
    - deployment
    - wait
    - installtest

### create an artifact with a pre-configured spack on each push
setup_spack_push:
    stage: prepare-spack
    tags:
        - k4-build-spack-release
        - k4-build-spack-nightlies
    only:
      refs:
          - pushes
          - merge_requests
          - schedules
          - web
    script:
        # log all variables
        - echo "SPACK_INSTALL_ARGUMENTS ... ${SPACK_INSTALL_ARGUMENTS}"
        - echo "K4_CI_URLS ... ${K4_CI_URLS}"
        - echo "K4_JOBTYPE ... ${K4_JOBTYPE}"
        - echo "SPACK_VERSION ... ${SPACK_VERSION}"
        # set up spack inside the k4-spack repo
        - source scripts/ci_setup_spack.sh

    artifacts:
        paths:
            - key4hep-spack.tar.gz
            - key4hep-spack_centos7-cvmfs.tar.gz
            - spack 
        expire_in: 1 month

#### CI builds of key4hep-stack
# This is intended to run more complex ci builds, possibly comprising PRs in several repos
# this job expects the following setup on the runner:
# * environment variables GITHUB_TOKEN and GITHUB_USER
#   set in .bashrc or via gitlab ci - to be able to query the commits on github
#  
build-spack-ci:
    stage: compilation
    tags:
        - cvmfs
    needs:
        - "setup_spack_push"
    parallel:
        matrix:
          - SPACKENV: 
            - key4hep-ci
    only:
      refs:
          - web
      variables:
          - $K4_JOBTYPE == "CI"
    script:
        - |
          #set up spack inside the k4-spack repo
          source spack/share/spack/setup-env.sh
          export KEY4HEP_RELEASE_VERSION=master-ci-`date -I`
          python scripts/fetch_nightly_versions.py >> environments/$SPACKENV/spack.yaml
          cd environments/$SPACKENV
          source setup_clingo_centos7.sh
          spack env activate .
          git clone https://gitlab.cern.ch/sft/sft-spack-repo.git
          spack repo add ./sft-spack-repo
          spack add key4hep-stack@${KEY4HEP_RELEASE_VERSION}
          for K4_CI_URL in $K4_CI_URLS; do
            # setup development repos
            export PR_NUMBER=`echo $K4_CI_URL | grep -oP "pull/\K[0-9]*"`
            export PR_REPO_URL=`echo $K4_CI_URL | grep -oP "\K.*/pull/"`
            export PR_REPO_URL=`echo ${PR_REPO_URL/\/pull\//}`
            export PACKAGE_NAME=`echo ${PR_REPO_URL} | rev | cut -d "/" -f -1 | rev `
            export PACKAGE_NAME=`echo $PACKAGE_NAME |  tr '[:upper:]' '[:lower:]'`
            echo "PACKAGE_NAME: ${PACKAGE_NAME}"
            echo "PR_NUMBER: ${PR_NUMBER}"
            echo "PR_REPO_URL: ${PR_REPO_URL}"
            #sed -i "/${PACKAGE_NAME}@commit/d" ./spack.yaml
            mkdir $PACKAGE_NAME; cd $PACKAGE_NAME
            git clone $PR_REPO_URL .
            #spack add ${PACKAGE_NAME}@commit.`git rev-parse HEAD`
            spack develop -p $PWD ${PACKAGE_NAME}@commit.`git rev-parse HEAD`
            git fetch origin pull/${PR_NUMBER}/head:testbranch${PR_NUMBER}
            git checkout testbranch${PR_NUMBER}
            cd ..
          done;
          # compile onwards and upwards
          spack install ${SPACK_INSTALL_ARGUMENTS}

#### Nightly build of key4hep-stack
# this job expects the following setup on the runner:
# * environment variables GITHUB_TOKEN and GITHUB_USER
#   set in .bashrc or via gitlab ci - to be able to query the commits on github
# *  (for the nightlies) existing installation of key4hep-stack (stable version) in /cvmfs/sw.hsf.org/spackages
#   (or whatever is defined in upstreams.yaml)
#   this may be read-only - could be delivered by cvmfs
# * writeable directory in /cvmfs/sw-nightlies.hsf.org 
#   (or whatever is defined in config.yaml)
#  
# for the cvmfs deployment:
# * the gitlab-runner user must be able to ssh into the cvmfs publisher
# * the cvmfs publisher must be able to rsync with the build machine
build-spack-nightlies:
    stage: compilation
    tags:
        - k4-build-spack-nightlies
    needs:
        - "setup_spack_push"
    parallel:
        matrix:
          - SPACKENV: 
            - key4hep-nightlies
            #- key4hep-nightlies-rootmod
            #- key4hep-nightlies-debug
    only:
      refs:
          - schedules # Only execute this on scheduled "nightly" pipelines
          - web
      variables:
          - $K4_JOBTYPE == "Nightlies"
    script:
        # set up spack inside the k4-spack repo
        - source spack/share/spack/setup-env.sh
        - export KEY4HEP_RELEASE_VERSION=master-`date -I`
        - if [ $SPACKENV == "key4hep-nightlies-rootmod" ]; then export KEY4HEP_RELEASE_VERSION=master-rootmod-`date -I`; fi
        - python scripts/fetch_nightly_versions.py >> environments/$SPACKENV/spack.yaml

        - cd environments/$SPACKENV
        # symlink to latest now handled by deploy step
        #- source pathsetup.sh
        - spack env activate .
        - git clone https://gitlab.cern.ch/sft/sft-spack-repo.git
        - spack repo add ./sft-spack-repo
        - spack add key4hep-stack@${KEY4HEP_RELEASE_VERSION}
        # compile onwards and upwards
        - spack install ${SPACK_INSTALL_ARGUMENTS}


### Test lcg release in spack  on the key4hep infrastructure
build-lcg-release:
    stage: compilation
    tags:
        - k4-build-spack-release
    needs:
        - "setup_spack_push"
    only:
      refs:
          - tags
          - schedules # Only execute this on scheduled "nightly" pipelines
          - web
      variables:
          - $K4_JOBTYPE == "LCG_Release"
    script:
        - source spack/share/spack/setup-env.sh
        - git clone https://gitlab.cern.ch/sft/sft-spack-repo.git
        - cd sft-spack-repo
        - spack env activate -d environments/lcg-release/
        - spack install ${SPACK_INSTALL_ARGUMENTS}

### Test deployment of lcg-release
deploy-cvmfs-lcg:
    stage: deployment
    tags:
        - k4-build-spack-release
    needs: ["build-lcg-release"]
    only:
      refs:
          - tags
          - schedules # Only execute this on scheduled "nightly" pipelines
          - web
      variables:
          - $K4_JOBTYPE == "LCG_Release"
    script:
        - ssh cvswhsforg@cvmfs-sw-hsf-org.cern.ch ' bash -c ./cvmfs_deploy.sh'

### deploy the nightlies to cvmfs
deploy-cvmfs-nightlies:
    stage: deployment
    needs: ["build-spack-nightlies"]
    tags:
        - sw-nightlies-hsf-org-cvmfs-publisher
    only:
      refs:
          - schedules # Only execute this on scheduled "nightly" pipelines
          - web
      variables:
          - $K4_JOBTYPE == "Nightlies"
    script:
        - cd scripts/deploy-cvmfs-nightlies
        - ./cvmfs_deploy.sh


#### Release build of key4hep-stack
# this job expects the following setup on the runner:

# * writeable directory in /cvmfs/sw.hsf.org 
#   (or whatever is defined in config.yaml)
#  
# for the cvmfs deployment:
# * the gitlab-runner user must be able to ssh into the cvmfs publisher
# * the cvmfs publisher must be able to rsync with the build machine
build-spack-release:
    stage: compilation
    needs: ["setup_spack_push"]
    parallel:
        matrix:
          - SPACKENV: 
            - key4hep-release
            #- key4hep-release-clang
            #- key4hep-debug
    tags:
        - k4-build-spack-release
    only:
      refs:
          - tags
          - schedules # Only execute this on scheduled pipelines
          - web
      variables:
          - $K4_JOBTYPE == "Release"
    script:
        # set up spack inside the k4-spack repo
        - source spack/share/spack/setup-env.sh
        # if this workflow has been started by a tag, use the tag version. if not, use todays date
        - if [ -z "$CI_COMMIT_TAG" ]; then export K4STACK_VERSION=`date -I`; else export K4STACK_VERSION=$CI_COMMIT_TAG;  fi 
        - if [ ! -z "$CI_BUILD_TAG" ]; then export K4STACK_VERSION=$CI_BUILD_TAG;  fi 
        - echo $K4STACK_VERSION
        # activate environment
        - cd environments/${SPACKENV}
        - spack env activate .
        - git clone https://gitlab.cern.ch/sft/sft-spack-repo.git
        - spack repo add ./sft-spack-repo
        # config debug printouts
        - spack config blame config # debug printout
        - spack config blame packages # debug printout
        # in order to set the right version for the key4hep-stack spec, we add it to the
        # environment now when we now it
        - spack add key4hep-stack@${K4STACK_VERSION}
        # compile onwards and upwards
        - spack install ${SPACK_INSTALL_ARGUMENTS}| tee -a key4hep-stack-install.log
        # note the path of the just installed package
        # so that downstream jobs can source its setup
        # in an environment, this command should just find the specs  
        # installed in this environment
        - spack find -p key4hep-stack@${K4STACK_VERSION} > spackfind.log
        - cat spackfind.log # debug printout
        # parse output of spack find
        # the last line should be, p.ex.
        # zlib@1.2.11  /cvmfs/sw.hsf.org/spackages/linux-ubuntu20.04-broadwell/gcc-9.3.0/zlib-1.2.11-emkoq3o2mtu377gy6plexktmkqchkgkb
        # cut can only be used to get the first field - do a double reverse of the string to get the last
        - SPACKENV_PATH=$(tail -n 1 spackfind.log  |  rev | cut -d " " -f1 | rev)
        - echo ${SPACKENV_PATH} # debug printout
        # we want to use the environment name as a variable name
        # so need to remove some special characters (right now only the hyphen)
        - echo ${SPACKENV//-/} # debug printout
        # make the path available to downstream jobs 
        # for example:
        #key4hepdebug_PATH=/cvmfs/sw.hsf.org/spackages2/key4hep-stack/2021-06-16/x86_64-centos7-gcc8.3.0-dbg/a7xzrrsha54w4mewe43b4bmxaqxnvg2x
        - echo "${SPACKENV//-/}_PATH=${SPACKENV_PATH}" >> build.env
        - cat build.env # debug printout
    artifacts:
        # the log could be saved as an artifact, but does not really have more info than stdout
        #paths:
        #  - environments/${SPACKENV}/key4hep-stack-install.log
        reports:
          dotenv: environments/${SPACKENV}/build.env


buildtest-spack-release:
    stage: buildtest
    needs:
        - "build-spack-release"
        - "setup_spack_push"
    parallel:
        matrix:
          - SPACKENV: 
            - key4hep-release
            #- key4hep-debug
    tags:
        - k4-build-spack-release
    only:
      refs:
          - tags
          - schedules # Only execute this on scheduled "nightly" pipelines
          - web
      variables:
          - $K4_JOBTYPE == "Release"
    script:
        - echo ${SPACKENV} # debug printout
        - ls environments/${SPACKENV} # debug printout
        # resolve dynamic variable name
        - K4_SETUP_PATH=${SPACKENV//-/}_PATH
        - echo ${!K4_SETUP_PATH} # debug printout
        # run tests, to fail early, even before publishing
        - source ${!K4_SETUP_PATH}/setup.sh
        - ./scripts/ci_install_tests.sh



deploy-cvmfs-release:
    stage: deployment
    needs: ["build-spack-release"]
    tags:
        - k4-build-spack-release
    only:
      refs:
          - tags
          - schedules # Only execute this on scheduled "nightly" pipelines
          - web
      variables:
          - $K4_JOBTYPE == "Release"
    script:
        # todo: once the publisher is set up as a gitlab runner, use the script from the repo
        - ssh cvswhsforg@cvmfs-sw-hsf-org.cern.ch ' bash -c ./cvmfs_deploy.sh'

wait-cvmfs-release:
    stage: wait
    needs:
        - "build-spack-release"
        - "setup_spack_push"
        - "deploy-cvmfs-release"
    timeout: 20m
    parallel:
        matrix:
          - SPACKENV: 
            - key4hep-release
            #- key4hep-debug
    tags:
        - cvmfs
    only:
      refs:
          - tags
          - schedules # Only execute this on scheduled "nightly" pipelines
          - web
      variables:
          - $K4_JOBTYPE == "Release"
    script:
        # resolve dynamic variable name
        - K4_LATEST_SETUP_PATH=${SPACKENV//-/}_PATH
        - echo ${!K4_LATEST_SETUP_PATH} # debug printout
        # wait for the new release to be present on cvmfs
        # both to be sure that it is available once the pipeline finishes
        # and so the final install test can use it to check the publication
        - while [ ! -f ${!K4_LATEST_SETUP_PATH}/setup.sh ]; do sleep 10; done

test-cvmfs-release:
    stage: installtest
    needs: ["setup_spack_push", "deploy-cvmfs-release", "build-spack-release", "wait-cvmfs-release"]
    parallel:
        matrix:
          - SPACKENV: 
            - key4hep-release
            #- key4hep-debug
    tags:
        - cvmfs
    only:
      refs:
          - tags
          - schedules # Only execute this on scheduled "nightly" pipelines
          - web
      variables:
          - $K4_JOBTYPE == "Release"
    script:
        # resolve dynamic variable name
        - K4_LATEST_SETUP_PATH=${SPACKENV//-/}_PATH
        - echo ${!K4_LATEST_SETUP_PATH} # debug printout
        - source ${!K4_LATEST_SETUP_PATH}/setup.sh
        - ./scripts/ci_install_tests.sh