.gitlab-ci.yml

# default installed image for docker executor is: python:3.6
# using an image that can do git, docker, docker-compose
image: gitlab-registry.mpcdf.mpg.de/nomad-lab/nomad-fair/ci-runner:latest

# build directory inside
# https://gitlab.mpcdf.mpg.de/help/ci/runners/configure_runners.md#custom-build-directories

# https://docs.gitlab.com/ee/ci/yaml/workflow.html
# https://docs.gitlab.com/ee/ci/variables/predefined_variables.html
# if: CI_COMMIT_BRANCH && CI_COMMIT_BEFORE_SHA == "0000000000000000000000000000000000000000"
# A branch pipeline, but it is the first commit for that branch
# if: $CI_COMMIT_BRANCH && $CI_OPEN_MERGE_REQUESTS && $CI_PIPELINE_SOURCE == "push"
# For an existing workflow section to switch from branch pipelines to merge request pipelines when a merge request is created.
# if: $CI_COMMIT_BRANCH && $CI_OPEN_MERGE_REQUESTS
# A branch pipeline, but a merge request is open for that branch, do not run the branch pipeline.
# if: $CI_PIPELINE_SOURCE == "merge_request_event"
# A merge request pipeline, start the pipeline.
# if: $CI_COMMIT_BRANCH
# A branch pipeline, but there is no merge request open for the branch, run the branch pipeline.

default:
  tags:
    # Necessary to select the right CI runner
    - cloud

variables:
  DOCKER_TAG: ${CI_COMMIT_REF_SLUG}
  DOCKER_HOST: tcp://docker-dind:2375
  DOCKER_TLS_CERTDIR: ""

workflow:
  rules:
    - if: $CI_PIPELINE_SOURCE == "merge_request_event"
    - if: $CI_PIPELINE_SOURCE == "schedule"
    - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH
    - if: $CI_COMMIT_TAG
      variables:
        DOCKER_TAG: ${CI_COMMIT_REF_NAME}
    - when: never

stages:
  - build
  - test
  - deploy
  - release

update changelog:
  stage: build
  script:
    - curl -X POST "https://gitlab.mpcdf.mpg.de/api/v4/projects/$CI_PROJECT_ID/repository/changelog?version=${CI_COMMIT_TAG:1}&access_token=$CI_ACCESS_TOKEN&message=Add%20changelog%20for%20version%20$CI_COMMIT_TAG%20%5Bskip-ci%5D"
  rules:
    - if: $CI_COMMIT_TAG =~ /^v\d+\.\d+\.\d+$/

.build_image:
  image:
    name: gcr.io/kaniko-project/executor:debug
    entrypoint: [""]
  before_script:
    - echo "{\"auths\":{\"${CI_REGISTRY}\":{\"auth\":\"$(printf "%s:%s" "${CI_REGISTRY_USER}" "${CI_REGISTRY_PASSWORD}" | base64 | tr -d '\n')\"},\"$CI_DEPENDENCY_PROXY_SERVER\":{\"auth\":\"$(printf "%s:%s" ${CI_DEPENDENCY_PROXY_USER} "${CI_DEPENDENCY_PROXY_PASSWORD}" | base64 | tr -d '\n')\"}}}" > /kaniko/.docker/config.json
  script:
    - /kaniko/executor
      --context "."
      --dockerfile "Dockerfile"
      --target "${TARGET}"
      --destination "${DESTINATION}"
      --cache-repo "${CI_REGISTRY_IMAGE}/cache"
      --cache=true
      --skip-unused-stages
      --build-arg SETUPTOOLS_SCM_PRETEND_VERSION

build gui:
  stage: build
  extends: .build_image
  variables:
    TARGET: dev_node
    DESTINATION: "${CI_REGISTRY_IMAGE}/dev_node:${DOCKER_TAG}"

# kaniko image doesn't contain pip, so we have to save the scm_pretend_version in an earlier job and reuse it later
update_scm_pretend_version:
  stage: build
  script:
    - pip install --upgrade setuptools_scm
    - echo "SETUPTOOLS_SCM_PRETEND_VERSION=$(python -m setuptools_scm)" >> build.env
  artifacts:
    reports:
      dotenv: build.env

build python:
  stage: build
  extends: .build_image
  variables:
    TARGET: dev_python
    DESTINATION: "${CI_REGISTRY_IMAGE}/dev_python:${DOCKER_TAG}"

python linting:
  stage: test
  image: ${CI_REGISTRY_IMAGE}/dev_python:${DOCKER_TAG}
  script:
    - ruff check nomad tests
    - ruff format nomad tests --check
    - ruff check nomad tests --output-format gitlab > $CI_PROJECT_DIR/gl-code-quality-report.json
    - mypy nomad tests
  rules:
    - if: $CI_COMMIT_TAG
      when: never
    - when: on_success
  artifacts:
    name: "nomad_code_quality"
    when: always
    reports:
      codequality: gl-code-quality-report.json
    expire_in: never

python package clean up:
  stage: test
  image: ${CI_REGISTRY_IMAGE}/dev_python:${DOCKER_TAG}
  script:
    - python scripts/cleanup_packages.py
  rules:
    - when: manual
      allow_failure: true

check python dependencies:
  stage: test
  image: python:3.9
  before_script:
    - pip install --upgrade uv
  script:
    - scripts/check_python_dependencies.sh
  rules:
    - if: $CI_COMMIT_TAG
      when: never
    - when: manual
      allow_failure: true

.base_test:
  image: ${CI_REGISTRY_IMAGE}/dev_python:${DOCKER_TAG}
  stage: test
  services:
    - name: rabbitmq:4
      alias: rabbitmq
    - name: elasticsearch:7.17.24
      alias: elastic
      command:
        - bash
        - "-c"
        - ES_JAVA_OPTS="-Xms512m -Xmx512m" docker-entrypoint.sh elasticsearch -Ediscovery.type=single-node -Expack.security.enabled=false
    - name: mongo:5
      alias: mongo
  variables:
    RABBITMQ_ERLANG_COOKIE: SWQOKODSQALRPCLNMEQG
    RABBITMQ_DEFAULT_USER: rabbitmq
    RABBITMQ_DEFAULT_PASS: rabbitmq
    RABBITMQ_DEFAULT_VHOST: /
    NOMAD_RABBITMQ_HOST: rabbitmq
    NOMAD_ELASTIC_HOST: elastic
    NOMAD_MONGO_HOST: mongo
    NOMAD_KEYCLOAK_PASSWORD: ${CI_KEYCLOAK_ADMIN_PASSWORD}
    NOMAD_NORMALIZE_SPRINGER_DB_PATH: /nomad/fairdi/db/data/springer.msg
  before_script:
    - scripts/check_elastic.sh
    - uv pip install -e ".[dev]"
    - uv pip install -r requirements-plugins.txt -c requirements-dev.txt # remove this after moving to distro

generate pytest timings:
  extends: .base_test
  script:
    - python -m pytest --store-durations
  artifacts:
    expire_in: 1 days
    when: on_success
    paths:
      - .test_durations

  rules:
    - if: $CI_COMMIT_TAG
      when: never
    - when: manual
      allow_failure: true

python tests:
  parallel: 3
  extends: .base_test
  script:
    - python -m pytest --cov=nomad --cov-report term --cov-config=.coveragerc -sv tests --splits $CI_NODE_TOTAL --group=$CI_NODE_INDEX
  after_script:
    - cp .coverage .coverage_${CI_NODE_INDEX}
  artifacts:
    expire_in: 1 days
    when: on_success
    paths:
      - .coverage_${CI_NODE_INDEX}
  rules:
    - if: $CI_COMMIT_TAG
      when: never
    - when: on_success

python coverage report:
  stage: test
  image: ${CI_REGISTRY_IMAGE}/dev_python:${DOCKER_TAG}
  needs: ["python tests"]
  script:
    - coverage combine .coverage_*
    - coverage report
    - coverage xml
  coverage: /(?i)total.*? (100(?:\.0+)?\%|[1-9]?\d(?:\.\d+)?\%)$/
  artifacts:
    when: always
    reports:
      coverage_report:
        coverage_format: cobertura
        path: coverage.xml
  rules:
    - if: $CI_COMMIT_TAG
      when: never
    - when: on_success

gui linting:
  stage: test
  image: ${CI_REGISTRY_IMAGE}/dev_node:${DOCKER_TAG}
  variables:
    GIT_STRATEGY: none
  before_script:
    - cd /app/gui
  script:
    - yarn run lint
  rules:
    - if: $CI_COMMIT_TAG
      when: never
    - when: on_success

gui tests:
  stage: test
  image: ${CI_REGISTRY_IMAGE}/dev_node:${DOCKER_TAG}
  variables:
    GIT_STRATEGY: none
  before_script:
    - cd /app/gui
  script:
    - yarn test --ci --collectCoverage --reporters=default --reporters=jest-junit --silent
      --testPathIgnorePatterns=src/components/entry/ArchiveEntryView.spec.js
      --testPathIgnorePatterns=src/components/archive/FileBrowser.spec.js
      --testPathIgnorePatterns=src/components/archive/MetainfoBrowser.spec.js

  after_script:
    - cd /app/gui
    - cp junit.xml $CI_PROJECT_DIR
    - cp coverage/cobertura-coverage.xml $CI_PROJECT_DIR
  timeout: 1h 30m
  coverage: /Lines\s*:\s*(\d+.?\d*)%/
  artifacts:
    when: always
    reports:
      junit:
        - junit.xml
      coverage_report:
        coverage_format: cobertura
        path: cobertura-coverage.xml
  rules:
    - if: $CI_COMMIT_TAG
      when: never
    - when: on_success

check gui artifacts:
  stage: test
  image: ${CI_REGISTRY_IMAGE}/dev_python:${DOCKER_TAG}
  script:
    - uv pip install ".[dev,parsing,infrastructure]"
    - uv pip install -r requirements-plugins.txt -c requirements-dev.txt # remove this after moving to distro
    - scripts/check_gui_artifacts.sh
  after_script:
    - >
      if [ $CI_JOB_STATUS == 'failed' ]; then
        echo 'Make sure that the right GUI artifacts have been regenerated and committed by running'
        echo './scripts/generate_gui_artifacts.sh'
      fi
  rules:
    - if: $CI_COMMIT_TAG
      when: never
    - when: on_success

build python package:
  stage: test
  variables:
    SETUPTOOLS_SCM_PRETEND_VERSION: "${SETUPTOOLS_SCM_PRETEND_VERSION}"
  extends: .build_image
  script:
    - /kaniko/executor
      --context "."
      --dockerfile "Dockerfile"
      --target "dev_package"
      --destination "${CI_REGISTRY_IMAGE}/dev_package:${DOCKER_TAG}"
      --skip-unused-stages
      --cache-repo "${CI_REGISTRY_IMAGE}/cache"
      --cache=true
      --build-arg SETUPTOOLS_SCM_PRETEND_VERSION
    - mkdir -p $CI_PROJECT_DIR/dist
    - cp -r /app/dist/* $CI_PROJECT_DIR/dist
    - cp /app/tests/data/parsers/archive.json $CI_PROJECT_DIR/
    - cp /app/tests/data/examples/example.out $CI_PROJECT_DIR/
  artifacts:
    expire_in: 1 days
    when: on_success
    paths:
      - dist/
      - archive.json
      - example.out

install tests:
  stage: test
  parallel:
    matrix:
      - PYTHON_VERSION: ["3.9", "3.10", "3.11"]
  image: python:${PYTHON_VERSION}
  needs: ["build python package"]
  variables:
    UV_SYSTEM_PYTHON: true
    GIT_STRATEGY: none
  before_script:
    - pip install --upgrade pip
    - pip install uv==0.5
  script:
    - pip install dist/nomad-*.tar.gz
    - python -c 'import nomad.cli'
    - python -c 'from nomad.client import ArchiveQuery'
    - python -m nomad.cli parse --skip-normalizers archive.json
    - uv pip install git+https://github.com/nomad-coe/nomad-parser-example.git@ba6027fdd4cda0cf9e0b32546bd809c8fdda79e6
    - python -m exampleparser example.out

python package:
  stage: release
  variables:
    GIT_STRATEGY: none
  before_script:
    - pip install twine
  script: twine upload -u gitlab-ci-token -p ${CI_JOB_TOKEN} --repository-url https://gitlab.mpcdf.mpg.de/api/v4/projects/${CI_PROJECT_ID}/packages/pypi dist/nomad-lab-*.tar.gz
  rules:
    - if: $CI_COMMIT_BRANCH == "develop" && $NIGHTLY
      when: on_success
    - when: manual
      allow_failure: true
    - if: $CI_COMMIT_TAG

pypi package:
  stage: release
  variables:
    GIT_STRATEGY: none
  before_script:
    - pip install twine
  script: twine upload -u $CI_TWINE_USER -p $CI_TWINE_PASSWORD dist/nomad-lab-*.tar.gz
  rules:
    - if: $CI_COMMIT_TAG
      when: manual
      allow_failure: true
    - when: never

push to github:
  stage: release
  script:
    - git checkout ${CI_COMMIT_REF_NAME}
    - git push "https://${CI_GITHUB_ACCESS_TOKEN}@github.com/nomad-coe/nomad.git" ${CI_COMMIT_REF_NAME}
  rules:
    - if: $CI_COMMIT_BRANCH == "develop" || $CI_COMMIT_TAG