.gitlab-ci.yml

include:
  - local: .gitlab/sanitizers-ci.yml
  - local: .gitlab/benchmarks-ci.yml
  - local: .gitlab/build-ont_core.yml

workflow:
  rules:
    # If a new commit is pushed to the branch but an MR is already opened, don't run a "push" pipeline because
    # the merge request train setting will trigger a "merge_request_event" pipeline anyway.
    # In all other cases, run the pipeline.
    - if: $CI_PIPELINE_SOURCE == 'push' && $CI_OPEN_MERGE_REQUESTS
      when: never
    - if: $CI_PIPELINE_SOURCE == 'merge_request_event'
    - if: $CI_PIPELINE_SOURCE == 'schedule'
    - if: $CI_PIPELINE_SOURCE == 'push'
    - if: $CI_PIPELINE_SOURCE == 'web'

default:
  interruptible: true

stages:
  - pre-flight
  - build
  - test
  - deploy
  - regression_test

variables:
  BUILD_DIR: build
  BUILD_TARGET: install
  BUILD_TYPE: Release
  GIT_SUBMODULE_STRATEGY: recursive
  CUDA: "11.8"
  WIN_CUDA_TOOLKIT: "/c/Program Files/NVIDIA GPU Computing Toolkit/CUDA/v${CUDA}"
  MTL_DEBUG_LAYER: 1

# Build stage components

.macos_arm_runner:
  tags:
    - osx_arm64
    - xcode-14.2
  variables:
    MACOSX_DEPLOYMENT_TARGET: "11.0"

.macos_x64_runner:
  tags:
    - osx_x86
    - xcode-14.2
  variables:
    MACOSX_DEPLOYMENT_TARGET: "10.15"

.linux_runner:
  tags:
    - ubuntu-20.04
    - nvidia-docker

.linux_arm64_runner:
  tags:
    - linux_aarch64
    - docker-arm

.linux_tegra_runner:
  tags:
    - nvidia-docker-tegra-gpu

.linux_orin_runner:
  tags:
    - linux-arm64-gpu
    - nvidia-docker

.windows_runner:
  tags:
    - windows-10
    - VS2019
    - cuda-${CUDA}

.linux_dependencies:
  before_script:
    - echo Dorado dependency stage
    - apt-get update && apt-get install -y --no-install-recommends samtools curl libhdf5-dev libssl-dev libzstd-dev libsz2 ccache automake
    - pip install gcovr

.set_parallel_jobs:
  - |
    if [[ $OSTYPE == 'darwin'* ]]; then
      PARALLEL_JOBS=$(( $(sysctl -n hw.physicalcpu) / 2 ))
    elif [[ $(arch) != 'aarch64' ]]; then
      PARALLEL_JOBS=$(( $(nproc) / 4 ))
    else
      PARALLEL_JOBS=8
    fi
    PARALLEL_JOBS=$(( 8 > PARALLEL_JOBS ? 8 : PARALLEL_JOBS ))
  - echo "PARALLEL_JOBS '${PARALLEL_JOBS}'"

.setup_ccache:
  - if [[ -d '/data/gitlab-ci-cache' ]] && [[ $OSTYPE != 'darwin'* ]] && [[ $OSTYPE != 'msys' ]]; then
  - export CCACHE_DIR=/data/gitlab-ci-cache
  - fi
  - ccache -s || true

.run_dorado_cmake_build:
  - echo Dorado build stage
  - !reference [.set_parallel_jobs]
  - !reference [.setup_ccache]
  - echo "BUILD_OPTIONS '${BUILD_OPTIONS}'"
  - cmake ${BUILD_OPTIONS} -S . -B ${BUILD_DIR} -DCMAKE_BUILD_TYPE=${BUILD_TYPE}
  - cmake --build ${BUILD_DIR} --config ${BUILD_TYPE} --target ${BUILD_TARGET} -j ${PARALLEL_JOBS}

.run_dorado_cmake_ctest:
  - echo Dorado test stage
  - ctest -C ${BUILD_TYPE} --test-dir ${BUILD_DIR} --output-on-failure --verbose

.run_dorado_cmake_cpack:
  - echo Dorado packaging stage
  - cpack --config ${BUILD_DIR}/CPackConfig.cmake

.build_and_test_dorado:
  stage: build
  script:
    - !reference [.run_dorado_cmake_build]
    - !reference [.run_dorado_cmake_ctest]
  artifacts:
    paths:
      - dist
      # Uploading these allows us to avoid checking out the repo in tests.
      - tests/*.sh
      - tests/*.bat
      - tests/data
      - tests/cram
    expire_in: 1 day

.build_dorado_archive:
  stage: build
  script:
    - !reference [.run_dorado_cmake_build]
    - !reference [.run_dorado_cmake_cpack]
  artifacts:
    paths:
      - archive
    expire_in: 1 day

.sign_dorado:macos:
  script:
    # Older versions of macOS (10.15) can't run notarytool through xcrun, but can make
    # use of the standalone notarytool executable if it's copied from a newer system.
    # See https://developer.apple.com/forums/thread/712665.
    - |
      if command -v notarytool > /dev/null; then
        NOTARYTOOL=(notarytool)
      else
        NOTARYTOOL=(xcrun notarytool)
      fi
    # We sign the executable as part of CPack's packaging steps, so the package that it
    # produces should be ready to be notarized. That can be done asynchronously, but
    # doing everything in one pass (--wait flag) saves complicating the jobs.
    - ${NOTARYTOOL[@]} submit archive/dorado-*.zip
      --apple-id ${APPLE_NOTARIZATION_ID}
      --password ${APPLE_NOTARIZATION_PASSWORD}
      --team-id ${APPLE_NOTARIZATION_TEAM}
      --progress --verbose --wait
    # If notarization fails you need to run `xcrun notarytool log <request id>` to get
    # information about what failed.
    #
    # Extract the binary and check that it's notarized, since notarytool doesn't appear
    # to return an error on failure.
    - rm -rf notarization_check
    - unzip archive/dorado-*.zip -d notarization_check
    - spctl -a -t install -vvvv notarization_check/dorado-*/bin/dorado
    #
    # No stapling for now since you can only staple packages and not executables, hence
    # we'd need to switch to a dmg instead and do something like:
    #- xcrun stapler staple dorado-*.dmg

# Run this job manually, but always run on nightly.
.on_nightly_or_manual:
  rules:
    - if: $CI_PIPELINE_SOURCE == "schedule"
    - if: $CI_PIPELINE_SOURCE != "schedule"
      when: manual
      # See https://forum.gitlab.com/t/specifying-a-pipeline-step-as-manual-causes-its-status-to-always-be-blocked/58333/3
      allow_failure: true

# Pre-flight stages

pre-commit:
  image: ${DORADO_PRECOMMIT}
  variables:
    GIT_SUBMODULE_STRATEGY: none
  stage: pre-flight
  script:
    - echo "Pipeline source is ${CI_PIPELINE_SOURCE}"
    - pip install pre-commit
    - pre-commit run --all-files
  tags:
    - linux

# Build stages

build:linux:x86:focal:install_deps:
  image: ${DORADO_DOCKER_ROOT}/dorado-no-deps-20.04-cuda-${CUDA}.0:1.0
  extends:
    - .linux_dependencies
    - .linux_runner
    - .build_and_test_dorado
    - .on_nightly_or_manual

build:linux:x86:focal:
  image: ${DORADO_DOCKER_ROOT}/dorado-deps-20.04-cuda-${CUDA}.0:1.1
  variables:
    BUILD_OPTIONS: "-DBUILD_KOI_FROM_SOURCE=ON -DGITLAB_CI_TOKEN=${CI_JOB_TOKEN} -DONT_MM2_EXE=ON"
  extends:
    - .linux_dependencies
    - .linux_runner
    - .build_and_test_dorado

build:linux:x86:jammy:
  image: ${DORADO_DOCKER_ROOT}/dorado-deps-22.04-cuda-${CUDA}.0:1.1
  variables:
    BUILD_OPTIONS: "-DBUILD_KOI_FROM_SOURCE=ON -DGITLAB_CI_TOKEN=${CI_JOB_TOKEN} -DONT_MM2_EXE=ON"
  extends:
    - .linux_dependencies
    - .linux_runner
    - .build_and_test_dorado

build:linux:x86:focal:debug:
  variables:
    BUILD_TYPE: "Debug"
  extends:
    - build:linux:x86:focal
  rules:
    - if: $CI_MERGE_REQUEST_EVENT_TYPE == "merged_result"

build:linux:x86:focal_coverage:
  stage: build
  image: ${DORADO_DOCKER_ROOT}/dorado-deps-20.04-cuda-${CUDA}.0:1.1
  variables:
    BUILD_OPTIONS: "-DGENERATE_TEST_COVERAGE=1"
    BUILD_TARGET: "dorado_test_coverage"
  extends:
    - .linux_dependencies
    - .linux_runner
    - .on_nightly_or_manual
  script:
    - !reference [.run_dorado_cmake_build]
  artifacts:
    paths:
      - build/dorado_test_coverage/*

build:linux:x86:focal:clang_tidy:
  stage: build
  image: ${DORADO_DOCKER_ROOT}/dorado-deps-20.04-cuda-${CUDA}.0:1.1
  extends:
    - .linux_runner
  script:
    - apt update && apt install -y clang clang-tidy
    - !reference [.set_parallel_jobs]
    - ./.gitlab/run_clang_tidy.sh -j ${PARALLEL_JOBS}

.build:linux:x86:focal:valgrind:
  stage: build
  image: ${DORADO_DOCKER_ROOT}/dorado-deps-20.04-cuda-${CUDA}.0:1.1
  variables:
    # valgrind can't cope with huge executables, so we have to go dynamic.
    BUILD_OPTIONS: "-D TRY_USING_STATIC_TORCH_LIB=NO"
  extends:
    - .linux_runner
  script:
    - !reference [.set_parallel_jobs]
    - apt update && apt install -y autoconf autotools-dev automake wget libc6-dbg
    # We need a recent valgrind that includes https://bugs.kde.org/show_bug.cgi?id=402833.
    - VALGRIND_VERSION="3.22.0"
    - wget https://sourceware.org/pub/valgrind/valgrind-${VALGRIND_VERSION}.tar.bz2
    - tar -xvf valgrind-${VALGRIND_VERSION}.tar.bz2
    - pushd valgrind-${VALGRIND_VERSION}
    - ./autogen.sh
    - ./configure
    - make -j ${PARALLEL_JOBS}
    - valgrind=$(pwd)/vg-in-place
    - popd
    # We need symbols otherwise the generated reports are useless.
    - export CFLAGS="${CFLAGS} -g"
    - export CXXFLAGS="${CXXFLAGS} -g"
    # Build dorado.
    - !reference [.run_dorado_cmake_build]
    # Run the ordinary tests directly under valgrind.
    # We don't use `ctest -VV -E smoke -T memcheck` since that requires additional setup.
    - ${valgrind} --error-exitcode=1 --track-origins=yes -- ./${BUILD_DIR}/tests/dorado_tests

# We have to split the tegra builds into separate build+test jobs since we can't
# test on the machines we build on.
build:linux:arm64:bionic:
  image: ${DORADO_DOCKER_ROOT}/dorado-l4t-r32.4.3-deps:1.3
  variables:
    BUILD_OPTIONS: "-DONT_MM2_EXE=ON"
  extends:
    - .build_and_test_dorado
    - .linux_arm64_runner
  script:
    # Replace the script stage with just the build.
    - !reference [.run_dorado_cmake_build]
    # Modify the test paths so that we can run the tests on a different machine.
    - sed -i "s#$(pwd)#_BUILD_FOLDER_#g" ${BUILD_DIR}/tests/CTestTestfile.cmake
  artifacts:
    paths:
      # Can't !reference here, but it would be:
      #- !reference [.build_and_test_dorado, artifacts, path]
      - dist
      - tests/*.sh
      - tests/*.bat
      - tests/data
      - tests/cram
      # Need to artifact parts of the build folder for the testing job.
      - ${BUILD_DIR}/CTestTestfile.cmake
      - ${BUILD_DIR}/tests

build:linux:arm64:bionic:unittests:
  image: ${DORADO_DOCKER_ROOT}/dorado-l4t-r32.4.3-deps:1.3
  stage: build
  extends:
    - .linux_tegra_runner
  script:
    # Fixup the tests so that we can run them.
    - sed -i "s#_BUILD_FOLDER_#$(pwd)#g" ${BUILD_DIR}/tests/CTestTestfile.cmake
    - !reference [.run_dorado_cmake_ctest]
  needs:
    - build:linux:arm64:bionic

build:linux:arm64:focal:
  image: ${DORADO_DOCKER_ROOT}/dorado-l4t-pytorch-r35.1.0-pth1.13-py3:1.0
  variables:
    BUILD_OPTIONS: "-DONT_MM2_EXE=ON"
  extends:
    - .linux_dependencies
    - .linux_arm64_runner
    - .build_and_test_dorado

build:linux:arm64:focal:no_ccache:
  variables:
    BUILD_OPTIONS: "-D DORADO_DISABLE_CCACHE=ON"
  extends:
    - build:linux:arm64:focal
    - .on_nightly_or_manual

build:linux:x86:focal_koi_download:
  image: ${DORADO_DOCKER_ROOT}/dorado-deps-20.04-cuda-${CUDA}.0:1.1
  extends:
    - .linux_runner
    - .build_and_test_dorado

build:linux:x86:focal_koi_download:no_ccache:
  variables:
    BUILD_OPTIONS: "-D DORADO_DISABLE_CCACHE=ON"
  extends:
    - build:linux:x86:focal_koi_download
    - .on_nightly_or_manual

build:windows:
  extends:
    - .windows_runner
    - .build_and_test_dorado
  before_script:
    - export CUDA_TOOLKIT=$(cygpath -u $(cygpath -d "${WIN_CUDA_TOOLKIT}"))
    - export BUILD_OPTIONS="-A x64 -T cuda=${CUDA} -DCUDAToolkit_ROOT=${CUDA_TOOLKIT} -D WIN_THREADS=4 -DBUILD_KOI_FROM_SOURCE=ON -DGITLAB_CI_TOKEN=${CI_JOB_TOKEN}"

build:windows:debug:
  variables:
    BUILD_TYPE: "Debug"
  extends:
    - build:windows
  rules:
    - if: $CI_MERGE_REQUEST_EVENT_TYPE == "merged_result"

build:windows_koi_download:
  extends:
    - .windows_runner
    - .build_and_test_dorado
  before_script:
    - export CUDA_TOOLKIT=$(cygpath -u $(cygpath -d "${WIN_CUDA_TOOLKIT}"))
    - export BUILD_OPTIONS="-A x64 -T cuda=${CUDA} -DCUDAToolkit_ROOT=${CUDA_TOOLKIT} -D WIN_THREADS=4"

build:macos:arm:
  variables:
    BUILD_OPTIONS: "-DONT_MM2_EXE=ON"
  extends:
    - .macos_arm_runner
    - .build_and_test_dorado

build:macos:arm:no_ccache:
  variables:
    BUILD_OPTIONS: "-D DORADO_DISABLE_CCACHE=ON"
  extends:
    - build:macos:arm
    - .on_nightly_or_manual

build:macos:x64:
  variables:
    BUILD_OPTIONS: "-DONT_MM2_EXE=ON"
  extends:
    - .macos_x64_runner
    - .build_and_test_dorado

build:macos:x64:no_ccache:
  variables:
    BUILD_OPTIONS: "-D DORADO_DISABLE_CCACHE=ON"
  extends:
    - build:macos:x64
    - .on_nightly_or_manual

build:ios:m1:
  variables:
    BUILD_TARGET: "ALL_BUILD"
    PLATFORM: "OS64"
    SDK_NAME: "iphoneos"
    DORADO_RUN_TESTS: "OFF"
  extends:
    - .macos_arm_runner
    - .build_and_test_dorado
  script:
    # If MACOSX_DEPLOYMENT_TARGET is set then Xcode will try to build for macOS.
    - unset MACOSX_DEPLOYMENT_TARGET
    - export SDKROOT=$(xcrun --sdk ${SDK_NAME} --show-sdk-path)
    - echo "SDKROOT=${SDKROOT}"
    - export BUILD_OPTIONS="
      -G Xcode
      -D CMAKE_TOOLCHAIN_FILE=$(pwd)/cmake/ios-cmake/ios.toolchain.cmake
      -D PLATFORM=${PLATFORM}
      -D ENABLE_BITCODE=OFF
      -D SKIP_HDF_PLUGINS=ON
      -D DEPLOYMENT_TARGET=16.0
      -D DORADO_DISABLE_DORADO=ON
      -D DORADO_RUN_TESTS=${DORADO_RUN_TESTS}
      -D CMAKE_XCODE_ATTRIBUTE_CODE_SIGNING_ALLOWED=NO"
    - !reference [.build_and_test_dorado, script]

build:ios_sim:m1:
  variables:
    PLATFORM: "SIMULATORARM64"
    SDK_NAME: "iphonesimulator"
    DORADO_RUN_TESTS: "ON"
  extends:
    # Note that we don't technically need CMAKE_XCODE_ATTRIBUTE_CODE_SIGNING_ALLOWED
    # disabled for simulator builds, but it doesn't hurt.
    - build:ios:m1

build_archive:linux:x86:focal:
  image: ${DORADO_DOCKER_ROOT}/dorado-deps-20.04-cuda-${CUDA}.0:1.1
  variables:
    BUILD_OPTIONS: "-DBUILD_KOI_FROM_SOURCE=ON -DGITLAB_CI_TOKEN=${CI_JOB_TOKEN}"
  extends:
    - .linux_runner
    - .build_dorado_archive
    - .on_nightly_or_manual

build_archive:linux:x86:cuda12:
  image: nvcr.io/nvidia/pytorch:24.06-py3
  variables:
    # Note: the linker flag is a workaround for the provided torch forgetting to link to cublas.
    BUILD_OPTIONS: "-D DORADO_LIBTORCH_DIR=/usr/local/lib/python3.10/dist-packages/torch -D CMAKE_EXE_LINKER_FLAGS:STRING='-Wl,--copy-dt-needed-entries'"
  extends:
    - .linux_dependencies
    - .linux_runner
    - .build_dorado_archive
    - .on_nightly_or_manual

build_archive:linux:arm64:bionic:
  image: ${DORADO_DOCKER_ROOT}/dorado-l4t-r32.4.3-deps:1.3
  extends:
    - .linux_arm64_runner
    - .build_dorado_archive
    - .on_nightly_or_manual

build_archive:linux:arm64:focal:
  image: ${DORADO_DOCKER_ROOT}/dorado-l4t-pytorch-r35.1.0-pth1.13-py3:1.0
  extends:
    - .linux_dependencies
    - .linux_arm64_runner
    - .build_dorado_archive
    - .on_nightly_or_manual

build_archive:linux:x86:centos7:
  image: ${DORADO_DOCKER_ROOT}/dorado-deps-centos7-cuda-${CUDA}.0:1.1
  variables:
    BUILD_OPTIONS: "-DBUILD_KOI_FROM_SOURCE=ON -DGITLAB_CI_TOKEN=${CI_JOB_TOKEN} -DDORADO_USING_OLD_CPP_ABI=ON -DDYNAMIC_HDF=ON"
  extends:
    - .linux_runner
    - .build_dorado_archive
    - .on_nightly_or_manual

build_archive:windows:
  before_script:
    - export CUDA_TOOLKIT=$(cygpath -u $(cygpath -d "${WIN_CUDA_TOOLKIT}"))
    - export BUILD_OPTIONS="-A x64 -T cuda=${CUDA} -DCUDAToolkit_ROOT=${CUDA_TOOLKIT} -DBUILD_KOI_FROM_SOURCE=ON -DGITLAB_CI_TOKEN=${CI_JOB_TOKEN}"
  extends:
    - .windows_runner
    - .build_dorado_archive
    - .on_nightly_or_manual

build_archive:macos:arm:
  extends:
    - .macos_arm_runner
    - .build_dorado_archive
    - .on_nightly_or_manual
  script:
    - !reference [.build_dorado_archive, script]
    - !reference [.sign_dorado:macos, script]

build_archive:macos:x64:
  extends:
    - .macos_x64_runner
    - .build_dorado_archive
    - .on_nightly_or_manual
  script:
    - !reference [.build_dorado_archive, script]
    # Signing on x64 disabled for now since our CI machines are on 10.15.
    # See https://developer.apple.com/forums/thread/712665.
    #- !reference [.sign_dorado:macos, script]

# Test stage components

.test_dorado:
  stage: test
  variables:
    GIT_STRATEGY: none
    MODEL: dna_r10.4.1_e8.2_400bps_hac@v4.1.0
    BATCH: 384
    MODEL_SPEED: hac
    MODEL_VERSION: v4.2.0
    NO_TEST_DORADO_CORRECT: 0
    NO_TEST_DUPLEX: 0
  script:
    # Do everything inside a newly created folder, since some runners only run
    # tests and hence never perform a git clean.
    - rm -rf test_archive && mkdir test_archive && pushd test_archive
    - mv ../tests ../dist .
    - bash ./tests/test_simple_basecaller_execution.sh ./dist/bin/dorado ${MODEL} ${BATCH}
    - bash ./tests/test_simple_auto_basecaller_execution.sh ./dist/bin/dorado ${MODEL_SPEED} ${MODEL_VERSION} ${BATCH}
    - bash ./tests/test_expected_logging.sh ./dist/bin/dorado ${MODEL} ${BATCH}
    - bash ./tests/test_dorado_correct.sh ./dist/bin/dorado
    - bash ./tests/test_dorado_correct_cram.sh ./dist/bin/dorado

.test_dorado_windows:
  stage: test
  variables:
    GIT_STRATEGY: none
  script:
    # Do everything inside a newly created folder, since some runners only run
    # tests and hence never perform a git clean.
    - rm -rf test_archive && mkdir test_archive && pushd test_archive
    - mv ../tests ../dist .
    - .\\tests\\test_simple_basecaller_execution.bat .\\dist\\bin\\dorado.exe

.test_archive:
  stage: test
  variables:
    GIT_STRATEGY: none
  script:
    # Do everything inside a newly created folder, since some runners only run
    # tests and hence never perform a git clean.
    - rm -rf test_archive && mkdir test_archive && pushd test_archive
    - mv ../archive .
    - tar -xzf archive/dorado-*.gz
    - ldd ./dorado-*/bin/dorado
    - if [ -e ./dorado-*/lib/libdorado_torch_lib.so ]; then ldd ./dorado-*/lib/libdorado_torch_lib.so; fi
    # We just want to check that dorado will run here, so simply display the verbose version info
    - ./dorado-*/bin/dorado -vv

# Test stages

test:windows:
  extends:
    - .windows_runner
    - .test_dorado_windows
  needs:
    - build:windows

test:linux:x86:focal:
  image: ${DORADO_DOCKER_ROOT}/dorado-deps-20.04-cuda-${CUDA}.0:1.1
  extends:
    - .linux_dependencies
    - .linux_runner
    - .test_dorado
  needs:
    - build:linux:x86:focal
  artifacts:
    when: on_failure
    paths:
      - tests
    expire_in: 1 day

test:linux:arm64:bionic:
  image: ${DORADO_DOCKER_ROOT}/dorado-l4t-r32.4.3-deps:1.3
  extends:
    - .linux_dependencies
    - .linux_tegra_runner
    - .test_dorado
  variables:
    GIT_STRATEGY: none
    MODEL: dna_r10.4.1_e8.2_400bps_hac@v4.1.0
    BATCH: 0
    MODEL_SPEED: hac
    MODEL_VERSION: v4.2.0
    NO_TEST_DORADO_CORRECT: 1
    NO_TEST_DUPLEX: 1
  needs:
    - build:linux:arm64:bionic

test:linux:arm64:focal:
  image: ${DORADO_DOCKER_ROOT}/dorado-l4t-pytorch-r35.1.0-pth1.13-py3:1.0
  extends:
    - .linux_dependencies
    - .linux_orin_runner
    - .test_dorado
  needs:
    - build:linux:arm64:focal

test:macos:arm:
  extends:
    - .macos_arm_runner
    - .test_dorado
  needs:
    - build:macos:arm

test:macos:x64:
  extends:
    - .macos_x64_runner
    - .test_dorado
  needs:
    - build:macos:x64
  variables:
    GIT_STRATEGY: none
    MODEL: dna_r10.4.1_e8.2_400bps_fast@v4.1.0
    BATCH: 0
    MODEL_SPEED: hac
    MODEL_VERSION: v4.2.0
    NO_TEST_DORADO_CORRECT: 1
    NO_TEST_DUPLEX: 1
  timeout: 1h 30m

# Test that you can run dorado in a clean cuda 20.04 environment
test_archive:linux:x86:20.04_nvidia:
  image: nvidia/cuda:${CUDA}.0-devel-ubuntu20.04
  extends:
    - .linux_runner
    - .test_archive
  needs:
    - build_archive:linux:x86:centos7

# Test that you can run dorado in a clean cpu 18.04 environment
test_archive:linux:x86:18.04:
  image: ${DORADO_DOCKER_EXT}/ubuntu:18.04
  extends:
    - .linux_runner
    - .test_archive
  needs:
    - build_archive:linux:x86:centos7

# Test that you can run dorado in a clean cpu 20.04 environment
test_archive:linux:x86:20.04:
  image: ${DORADO_DOCKER_EXT}/ubuntu:20.04
  extends:
    - .linux_runner
    - .test_archive
  needs:
    - build_archive:linux:x86:centos7

# Test that you can run dorado in a clean l4t environment
test_archive:linux:arm64:18.04:
  image: nvcr.io/nvidia/l4t-base:r32.4.3
  extends:
    - .linux_tegra_runner
    - .test_archive
  needs:
    - build_archive:linux:arm64:bionic

test_archive:linux:arm64:20.04:
  image: ${DORADO_DOCKER_ROOT}/dorado-l4t-pytorch-r35.1.0-pth1.13-py3:1.0
  extends:
    - .linux_dependencies
    - .linux_orin_runner
    - .test_archive
  needs:
    - build_archive:linux:arm64:focal

# Test that models can be downloaded and run.
test:model_download:linux:x86:focal:
  image: ${DORADO_DOCKER_ROOT}/dorado-deps-20.04-cuda-${CUDA}.0:1.1
  extends:
    - .linux_dependencies
    - .linux_runner
    - .on_nightly_or_manual
  stage: test
  needs:
    - build:linux:x86:focal
  script:
    - ./tests/test_model_download.py --exe ./dist/bin/dorado --data ./tests/data --batchsize 64

test:model_download:macos:arm:
  extends:
    - .macos_arm_runner
    - .on_nightly_or_manual
  stage: test
  needs:
    - build:macos:arm
  script:
    - ./tests/test_model_download.py --exe ./dist/bin/dorado --data ./tests/data
  timeout: 1h 30m

# Auto bachsize stats generation
generate_auto_benchmarks:linux:x86:focal:
  image: ${DORADO_DOCKER_ROOT}/dorado-deps-20.04-cuda-${CUDA}.0:1.1
  extends:
    - .linux_dependencies
  tags:
    - ubuntu-20.04
    - nvidia-docker
    - ${GPU}
  when: manual
  stage: test
  needs:
    - build:linux:x86:focal
  script:
    - nvidia-smi
    - ./benchmark/generate_chunk_auto_batchsize_benchmarks.sh ./dist/bin/dorado cuda:0
  artifacts:
    paths:
      - "*.cpp"
      - "*.h"
      - "*.csv"
    expire_in: 1 day
  parallel:
    matrix:
       - GPU: "gpu-a100"
       - GPU: "gpu-v100"
       - GPU: "gpu-a6000"
  timeout: 3h

# MLHub

.mlhub:
  stage: test
  when: manual
  image: ${TRIGGER_IMAGE}
  needs: []
  variables:
    GIT_STRATEGY: none

# MLHub - Single read eval
.mlhub_sre:
  extends:
    - .mlhub
  needs:
    - job: build:linux:x86:focal
      artifacts: false
  script:
    - echo ${MODELS}
    - |
      curl -i --header "Content-Type: application/json" \
      --request POST \
      --data '{
          "key": "'${MLHUB_TRIGGER_SRE}'",
          "job_name": "Dorado SRE: '${CONDITION}' '${CI_COMMIT_REF_NAME}' - '"$CI_COMMIT_TITLE"' ",
          "script_parameters": {
              "models":'${MODELS}',
              "duplex_models":'${DUPLEX_MODELS}',
              "sre_output":"'${SRE_OUTPUT_ROOT}'/'${CI_COMMIT_SHA}'/",
              "sre_config":"${projectDir}/configs/'${SRE_CONFIG}'",
              "basecaller_mode":"dorado_artifact",
              "basecaller":"'${CI_COMMIT_SHA}'"
              }
      }' \
      ${MLHUB_TRIGGER_URL}

mlhub:sre_4k_260bps:
  extends: .mlhub_sre
  variables:
    CONDITION: "4k_260bps"
    SRE_CONFIG: "dna_r10.4.1-e8.2-4khz-260bps-prom.json"
    MODELS: '["dna_r10.4.1_e8.2_260bps_fast@v4.1.0","dna_r10.4.1_e8.2_260bps_hac@v4.1.0","dna_r10.4.1_e8.2_260bps_sup@v4.1.0"]'
    DUPLEX_MODELS: "[]"

mlhub:sre_4k_400bps_v4_1:
  extends: .mlhub_sre
  variables:
    CONDITION: "4k_400bps"
    SRE_CONFIG: "dna_r10.4.1-e8.2-4khz-400bps-prom.json"
    MODELS: '["dna_r10.4.1_e8.2_400bps_fast@v4.1.0","dna_r10.4.1_e8.2_400bps_hac@v4.1.0","dna_r10.4.1_e8.2_400bps_sup@v4.1.0"]'
    DUPLEX_MODELS: "[]"

mlhub:sre_5k_400bps_v4_3:
  extends: .mlhub_sre
  variables:
    CONDITION: "5k_400bps"
    SRE_CONFIG: "dna_r10.4.1-e8.2-5khz-400bps-prom.json"
    MODELS: '["dna_r10.4.1_e8.2_400bps_fast@v4.3.0","dna_r10.4.1_e8.2_400bps_hac@v4.3.0","dna_r10.4.1_e8.2_400bps_sup@v4.3.0"]'
    DUPLEX_MODELS: "[]"

mlhub:sre_5k_400bps_v5:
  extends: .mlhub_sre
  variables:
    CONDITION: "5k_400bps"
    SRE_CONFIG: "dna_r10.4.1-e8.2-5khz-400bps-prom.json"
    MODELS: '["dna_r10.4.1_e8.2_400bps_fast@v5.0.0","dna_r10.4.1_e8.2_400bps_hac@v5.0.0","dna_r10.4.1_e8.2_400bps_sup@v5.0.0"]'
    DUPLEX_MODELS: "[]"

mlhub:sre_5k_400bps_duplex:
  extends: .mlhub_sre
  variables:
    CONDITION: "5k_400bps_duplex"
    SRE_CONFIG: "dna_r10.4.2-e8.2-5khz-400bps-prom.json"
    MODELS: '["dna_r10.4.1_e8.2_400bps_hac@v5.0.0","dna_r10.4.1_e8.2_400bps_sup@v5.0.0"]'
    DUPLEX_MODELS: '["dna_r10.4.1_e8.2_5khz_stereo@v1.3"]'

# MLHub - Remora
mlhub:remora-validate:
  extends:
    - .mlhub
  script:
    - |
      curl --header "Content-Type: application/json" \
      --request POST \
      --data '{
          "key": "'${MLHUB_TRIGGER_KEY_REMORA}'",
          "job_name": "Dorado Remora: '${CI_COMMIT_REF_NAME}' - '"$CI_COMMIT_TITLE"' ",
          "script_parameters": {
            "dorado_branch":"'${CI_COMMIT_SHA}'",
            "dorado_build_options":"",
            "dorado_build_threads":"-- -j 8",
            "dorado_device":"'${DEVICE}'",
            "dorado_model":"dna_r10.4.1_e8.2_400bps_hac@v4.0.0",
            "dorado_modbase_model":"dna_r10.4.1_e8.2_400bps_hac@v4.0.0_5mCG_5hmCG@v2",
            "remora_model":"/media/groups/machine_learning/active/rharris/shared/nf-dorado-remora-analysis_datasets/dna_r10.4.1_e8.2_400bps_hac_v4.0.0_5hmc_5mc_CG_v2.pt"
          }
      }' \
      ${MLHUB_TRIGGER_URL}
  parallel:
    matrix:
      - DEVICE:
          - "cuda:all"
          - "cpu"

# MLHub - Dorado integration test
mlhub:sweeper:
  extends:
    - .mlhub
  script:
    - |
      curl --header "Content-Type: application/json" \
      --request POST \
      --data '{
          "key": "'${MLHUB_TRIGGER_KEY_SWEEPER}'",
          "job_name": "dorado-sweeper : '${CI_COMMIT_REF_NAME}' - '${CI_COMMIT_SHORT_SHA}'",
          "script_parameters": {
            "dorado_branch": "'${CI_COMMIT_SHA}'",
            "dorado_exe": ""
          }
      }' \
      ${MLHUB_TRIGGER_URL}

# MLHub - Dorado integration test
mlhub:dorado-integration-tests:
  extends:
    - .mlhub
    - .on_nightly_or_manual
  script:
    - |
      curl --header "Content-Type: application/json" \
      --request POST \
      --data '{
          "key": "'${MLHUB_TRIGGER_KEY_DORADO_INTEGRATION_TEST}'",
          "job_name": "dorado-integration-test : '${CI_COMMIT_REF_NAME}'",
          "script_parameters": {
            "dorado_branch":"'${CI_COMMIT_SHA}'",
            "dorado_build_options":"",
            "prompt_clean_basecalls": true,
            "dorado_exe": "",
            "config": "'${DORADO_INTEGRATION_TEST_MLHUB_CONFIG}'"
          }
      }' \
      ${MLHUB_TRIGGER_URL}

# Deploy

deploy:all:
  stage: deploy
  variables:
    GIT_STRATEGY: none
  script:
    - ls -lh archive
  extends:
    - .on_nightly_or_manual
  artifacts:
    paths:
      - archive
    expire_in: 1 day
  needs:
    - build_archive:linux:x86:centos7
    - build_archive:linux:arm64:bionic
    - build_archive:linux:arm64:focal
    - build_archive:macos:arm
    - build_archive:macos:x64
    - build_archive:windows