self-hosted.yml

name: Self-hosted CI

on:
  pull_request:
    branches: [ master ]
    paths-ignore:
      - '**.md'
  workflow_dispatch:

jobs:

  make-tests:
    name: Make tests
    runs-on: [self-hosted, colord]
    env: 
      EXE: ./bin/colord
    steps:
    - name: Checkout repository and submodules
      uses: actions/checkout@v2
      with:
        submodules: recursive
           
    - name: make (g++ 9)
      run: |
        make -j32 CXX=g++-9
        make clean
        
    - name: make (g++ 10)
      run: |
        make -j32 CXX=g++-10
        make clean
        
    - name: make (g++ 11)
      run: |
        make -j32 CXX=g++-11
        make clean
        
    - name: make (default)
      run: |
        make -j32
      
    - name: Print usage
      run: |
        $EXE --help
    
  ########################################################################################     
    
  basic-tests:
    name: Basic tests
    runs-on: [self-hosted, colord]
    needs: make-tests
    env: 
      EXE: ./bin/colord
      API_EXE: ./bin/api_example
      ONT: ./test/M.bovis.fastq
      ONT_LOSSY: ./test/M.bovis.fastq.quan
      HIFI: ./test/D.melanogaster.fastq
      HIFI_LOSSY: ./test/D.melanogaster.fastq.quan
      CLR: ./test/A.thaliana.fastq
      CLR_LOSSY: ./test/A.thaliana.fastq.quan
 
    steps:
   
   
    - name: ONT default
      run: |
        $EXE compress-ont $ONT ont.default
        $EXE decompress ont.default ont.default.fastq
        cmp $ONT_LOSSY ont.default.fastq
        
    - name: HiFi default
      run: |
        $EXE compress-pbhifi $HIFI hifi.default
        $EXE decompress hifi.default hifi.default.fastq
        cmp $HIFI_LOSSY hifi.default.fastq
     
    - name: CLR default
      run: |
        $EXE compress-pbraw $CLR clr.default
        $EXE decompress clr.default clr.default.fastq
        cmp $CLR_LOSSY clr.default.fastq
        
    - name: ONT lossless
      run: |
        $EXE compress-ont --qual org $ONT ont.lossless
        $EXE decompress ont.lossless ont.lossless.fastq
        cmp $ONT ont.lossless.fastq
        
    - name: CLR lossless
      run: |
        $EXE compress-pbraw -q org $CLR clr.lossless
        $EXE decompress clr.lossless clr.lossless.fastq
        cmp $CLR clr.lossless.fastq
        
    - name: HiFi balanced
      run: |
        $EXE compress-pbhifi --priority balanced $HIFI hifi.balanced
        $EXE decompress hifi.balanced hifi.balanced.fastq
        cmp $HIFI_LOSSY hifi.balanced.fastq
        
    - name: CLR ratio
      run: |
        $EXE compress-pbraw --priority ratio $CLR clr.ratio
        $EXE decompress clr.ratio clr.ratio.fastq
        cmp $CLR_LOSSY clr.ratio.fastq
        
    - name: ONT reference based
      run: |
        $EXE compress-ont -G ./test/M.bovis-reference.fna -s $ONT ont.refbased
        $EXE decompress ont.refbased ont.refbased.fastq
        cmp $ONT_LOSSY ont.refbased.fastq
    
        
    - name: Archives information
      run: |
        $EXE info ont.default
        $EXE info hifi.default
        $EXE info clr.default 
        $EXE info ont.lossless 
        $EXE info hifi.balanced
        $EXE info clr.ratio    

    - name: ONT default (API)
      run: |
        $API_EXE ont.default > ont.default.api
        cmp $ONT_LOSSY ont.default.api
        
    - name: HiFi balanced (API)
      run: |
        $API_EXE hifi.balanced > hifi.balanced.api
        cmp $HIFI_LOSSY hifi.balanced.api
        
    - name: ONT reference-based (API)
      run: |
        $API_EXE ont.refbased > ont.refbased.api
        cmp $ONT_LOSSY ont.refbased.api
        
    - name: CLR lossless (API)
      run: |
        $API_EXE clr.lossless > clr.lossless.api
        cmp $CLR clr.lossless.api

 #########################################################################################   

  ont-ref-tests:
    needs: basic-tests
    name: ONT tests (reference-based)
    runs-on: [self-hosted, colord]
    env: 
      EXE: ./bin/colord
      DATA: ../../../../data
      OUT: ../../../../out
    
    steps:
     
    - name: Mb1-guppy (embedded reference)
      run: |
        $EXE compress-ont -t 32 -G ./test/M.bovis-reference.fna -s ${DATA}/Mb1-guppy.fastq ${OUT}/Mb1-guppy.lossy.withref
        $EXE decompress ${OUT}/Mb1-guppy.lossy.withref ${OUT}/Mb1-guppy.lossy.withref.fastq
        cmp ${DATA}/Mb1-guppy.fastq.quan ${OUT}/Mb1-guppy.lossy.withref.fastq
        
    - name: Mb1-guppy (embedded reference, lossless)
      run: |
        $EXE compress-ont -t 32 --qual org -G ./test/M.bovis-reference.fna -s ${DATA}/Mb1-guppy.fastq ${OUT}/Mb1-guppy.lossless.withref
        $EXE decompress ${OUT}/Mb1-guppy.lossless.withref ${OUT}/Mb1-guppy.lossless.withref.fastq
        cmp ${DATA}/Mb1-guppy.fastq ${OUT}/Mb1-guppy.lossless.withref.fastq 
        
    - name: Mb1-guppy (external reference)
      run: |
        $EXE compress-ont -t 32 -G ./test/M.bovis-reference.fna ${DATA}/Mb1-guppy.fastq ${OUT}/Mb1-guppy.lossy.noref
        $EXE decompress -G ./test/M.bovis-reference.fna ${OUT}/Mb1-guppy.lossy.noref ${OUT}/Mb1-guppy.lossy.noref.fastq
        cmp ${DATA}/Mb1-guppy.fastq.quan ${OUT}/Mb1-guppy.lossy.noref.fastq
        
    - name: Mb1-guppy (external reference, lossless)
      run: |
        $EXE compress-ont -t 32 --qual org -G ./test/M.bovis-reference.fna ${DATA}/Mb1-guppy.fastq ${OUT}/Mb1-guppy.lossless.noref
        $EXE decompress -G ./test/M.bovis-reference.fna ${OUT}/Mb1-guppy.lossless.noref ${OUT}/Mb1-guppy.lossless.noref.fastq
        cmp ${DATA}/Mb1-guppy.fastq ${OUT}/Mb1-guppy.lossless.noref.fastq 
        
     #########################################################################################    
    
  ont-tests:
    needs: basic-tests
    name: ONT tests
    runs-on: [self-hosted, colord]
    env: 
      EXE: ./bin/colord
      DATA: ../../../../data
      OUT: ../../../../out
      
    steps:
    
    - name: Mb1-guppy 
      run: |
        $EXE compress-ont -t 32 ${DATA}/Mb1-guppy.fastq ${OUT}/Mb1-guppy.lossy
        $EXE decompress ${OUT}/Mb1-guppy.lossy ${OUT}/Mb1-guppy.lossy.fastq
        cmp ${DATA}/Mb1-guppy.fastq.quan ${OUT}/Mb1-guppy.lossy.fastq  
        
    - name: Mb1-guppy (balanced)
      run: |
        $EXE compress-ont -t 32 --priority balanced ${DATA}/Mb1-guppy.fastq ${OUT}/Mb1-guppy.lossy.balanced
        $EXE decompress ${OUT}/Mb1-guppy.lossy.balanced ${OUT}/Mb1-guppy.lossy.balanced.fastq
        cmp ${DATA}/Mb1-guppy.fastq.quan ${OUT}/Mb1-guppy.lossy.balanced.fastq 
        
    - name: Mb1-guppy (ratio)
      run: |
        $EXE compress-ont -t 32 --priority ratio ${DATA}/Mb1-guppy.fastq ${OUT}/Mb1-guppy.lossy.ratio
        $EXE decompress ${OUT}/Mb1-guppy.lossy.ratio ${OUT}/Mb1-guppy.lossy.ratio.fastq
        cmp ${DATA}/Mb1-guppy.fastq.quan ${OUT}/Mb1-guppy.lossy.ratio.fastq 
      
    - name: Mb1-guppy (lossless)
      run: |
        $EXE compress-ont -t 32 -q org ${DATA}/Mb1-guppy.fastq ${OUT}/Mb1-guppy.lossless
        $EXE decompress ${OUT}/Mb1-guppy.lossless ${OUT}/Mb1-guppy.lossless.fastq
        cmp ${DATA}/Mb1-guppy.fastq ${OUT}/Mb1-guppy.lossless.fastq 
        
    - name: Mb1-guppy (lossless, balanced)
      run: |
        $EXE compress-ont -t 32 -q org --priority balanced ${DATA}/Mb1-guppy.fastq ${OUT}/Mb1-guppy.lossless.balanced
        $EXE decompress ${OUT}/Mb1-guppy.lossless.balanced ${OUT}/Mb1-guppy.lossless.balanced.fastq
        cmp ${DATA}/Mb1-guppy.fastq ${OUT}/Mb1-guppy.lossless.balanced.fastq 
        
    - name: Mb1-guppy (lossless, ratio)
      run: |
        $EXE compress-ont -t 32 -q org --priority ratio ${DATA}/Mb1-guppy.fastq ${OUT}/Mb1-guppy.lossless.ratio
        $EXE decompress ${OUT}/Mb1-guppy.lossless.ratio ${OUT}/Mb1-guppy.lossless.ratio.fastq
        cmp ${DATA}/Mb1-guppy.fastq ${OUT}/Mb1-guppy.lossless.ratio.fastq 
        
    #########################################################################################    
    
  hifi-tests:
    needs: basic-tests
    name: HiFi tests
    runs-on: [self-hosted, colord]
    env: 
      EXE: ./bin/colord
      DATA: ../../../../data
      OUT: ../../../../out
      
    steps:

    - name: ATCC 
      run: |
        $EXE compress-pbhifi -t 32 ${DATA}/ATCC.fastq ${OUT}/ATCC.lossy
        $EXE decompress ${OUT}/ATCC.lossy ${OUT}/ATCC.lossy.fastq
        cmp ${DATA}/ATCC.fastq.quan ${OUT}/ATCC.lossy.fastq  
        
    - name: ATCC (balanced)
      run: |
        $EXE compress-pbhifi -t 32 --priority balanced ${DATA}/ATCC.fastq ${OUT}/ATCC.lossy.balanced
        $EXE decompress ${OUT}/ATCC.lossy.balanced ${OUT}/ATCC.lossy.balanced.fastq
        cmp ${DATA}/ATCC.fastq.quan ${OUT}/ATCC.lossy.balanced.fastq 
        
    - name: ATCC (ratio)
      run: |
        $EXE compress-pbhifi -t 32 --priority ratio ${DATA}/ATCC.fastq ${OUT}/ATCC.lossy.ratio
        $EXE decompress ${OUT}/ATCC.lossy.ratio ${OUT}/ATCC.lossy.ratio.fastq
        cmp ${DATA}/ATCC.fastq.quan ${OUT}/ATCC.lossy.ratio.fastq 
      
    - name: ATCC (lossless)
      run: |
        $EXE compress-pbhifi -t 32 -q org ${DATA}/ATCC.fastq ${OUT}/ATCC.lossless
        $EXE decompress ${OUT}/ATCC.lossless ${OUT}/ATCC.lossless.fastq
        cmp ${DATA}/ATCC.fastq ${OUT}/ATCC.lossless.fastq 
        
    - name: ATCC (lossless, balanced)
      run: |
        $EXE compress-pbhifi -t 32 -q org --priority balanced ${DATA}/ATCC.fastq ${OUT}/ATCC.lossless.balanced
        $EXE decompress ${OUT}/ATCC.lossless.balanced ${OUT}/ATCC.lossless.balanced.fastq
        cmp ${DATA}/ATCC.fastq ${OUT}/ATCC.lossless.balanced.fastq 
        
    - name: ATCC (lossless, ratio)
      run: |
        $EXE compress-pbhifi -t 32 -q org --priority ratio ${DATA}/ATCC.fastq ${OUT}/ATCC.lossless.ratio
        $EXE decompress ${OUT}/ATCC.lossless.ratio ${OUT}/ATCC.lossless.ratio.fastq
        cmp ${DATA}/ATCC.fastq ${OUT}/ATCC.lossless.ratio.fastq 
        

#########################################################################################    
    
  clr-tests:
    needs: basic-tests
    name: CLR tests
    runs-on: [self-hosted, colord]
    env: 
      EXE: ./bin/colord
      DATA: ../../../../data
      OUT: ../../../../out
      
    steps:

    - name: yeast 
      run: |
        $EXE compress-pbraw -t 32 ${DATA}/yeast.fastq ${OUT}/yeast.lossy
        $EXE decompress ${OUT}/yeast.lossy ${OUT}/yeast.lossy.fastq
        cmp ${DATA}/yeast.fastq.quan ${OUT}/yeast.lossy.fastq  
        
    - name: yeast (balanced)
      run: |
        $EXE compress-pbraw -t 32 --priority balanced ${DATA}/yeast.fastq ${OUT}/yeast.lossy.balanced
        $EXE decompress ${OUT}/yeast.lossy.balanced ${OUT}/yeast.lossy.balanced.fastq
        cmp ${DATA}/yeast.fastq.quan ${OUT}/yeast.lossy.balanced.fastq 
        
    - name: yeast (ratio)
      run: |
        $EXE compress-pbraw -t 32 --priority ratio ${DATA}/yeast.fastq ${OUT}/yeast.lossy.ratio
        $EXE decompress ${OUT}/yeast.lossy.ratio ${OUT}/yeast.lossy.ratio.fastq
        cmp ${DATA}/yeast.fastq.quan ${OUT}/yeast.lossy.ratio.fastq 
        
#########################################################################################    
    
  big-tests:
    needs: [ont-ref-tests, ont-tests, hifi-tests, clr-tests]
    name: Exhaustive tests
    runs-on: [self-hosted, colord]
    env: 
      EXE: ./bin/colord
      DATA: ../../../../data
      OUT: ../../../../out
      
    steps:      
        
    - name: lun
      run: |
        $EXE compress-ont -t 32 ${DATA}/lun ${OUT}/lun.lossy
        $EXE decompress ${OUT}/lun.lossy ${OUT}/lun.lossy.fastq
        cmp ${DATA}/lun.quan ${OUT}/lun.lossy.fastq
        
    - name: lun (balanced)
      run: |
        $EXE compress-ont -t 32 --priority balanced ${DATA}/lun ${OUT}/lun.lossy.balanced
        $EXE decompress ${OUT}/lun.lossy.balanced ${OUT}/lun.lossy.balanced.fastq
        cmp ${DATA}/lun.quan ${OUT}/lun.lossy.balanced.fastq 
        
    - name: lun (ratio)
      run: |
        $EXE compress-ont -t 32 --priority ratio ${DATA}/lun ${OUT}/lun.lossy.ratio
        $EXE decompress ${OUT}/lun.lossy.ratio ${OUT}/lun.lossy.ratio.fastq
        cmp ${DATA}/lun.quan ${OUT}/lun.lossy.ratio.fastq 

    - name: lun (lossless)
      run: |
        $EXE compress-ont -t 32 -q org ${DATA}/lun ${OUT}/lun.lossless
        $EXE decompress ${OUT}/lun.lossless ${OUT}/lun.lossless.fastq
        cmp ${DATA}/lun ${OUT}/lun.lossless.fastq 
        
    - name: lun (lossless, balanced)
      run: |
        $EXE compress-ont -t 32 -q org --priority balanced ${DATA}/lun ${OUT}/lun.lossless.balanced
        $EXE decompress ${OUT}/lun.lossless.balanced ${OUT}/lun.lossless.balanced.fastq
        cmp ${DATA}/lun ${OUT}/lun.lossless.balanced.fastq 
        
    - name: lun (lossless, ratio)
      run: |
        $EXE compress-ont -t 32 -q org --priority ratio ${DATA}/lun ${OUT}/lun.lossless.ratio
        $EXE decompress ${OUT}/lun.lossless.ratio ${OUT}/lun.lossless.ratio.fastq
        cmp ${DATA}/lun ${OUT}/lun.lossless.ratio.fastq