application.conf

include  required(classpath("application"))

system.max-concurrent-workflows = 3
system.new-workflow-poll-rate = 10
system.io {
  number-of-attempts = 50
  }

backend {
  default = "PBSPRO"
  providers {
    PBSPRO {
      actor-factory = "cromwell.backend.impl.sfs.config.ConfigBackendLifecycleActorFactory"
      config {

        concurrent-job-limit = 100
        
        runtime-attributes = """
        String? docker
        String  walltime = "240:00:00"
        String  queue = "cgsd"
        Int     cpu = 6
        Int     memory_gb = 12
        """
        
        submit = """
        qsub \
        -N ${job_name} \
        -l walltime=${walltime} \
        -q ${queue} \
        -l ncpus=${cpu} \
        -l mem=${memory_gb}gb \
        -o ${out} \
        -e ${err} \
        ${script}
        """
        
        submit-docker = """
               
        module load singularity/3.7.2

          
        # mkdir -p /groups/cgsd/alexandre/.singularity/cache
        export SINGULARITY_CACHEDIR=$HOME/.singularity/cache
        
        # Singularity cachedir check if it exists, if not create one        
        
        if [ -z $SINGULARITY_CACHEDIR ];
          then CACHE_DIR=$HOME/.singularity/cache
          else CACHE_DIR=$HOME/.singularity/cache
        fi

        # make sure cache dir exists, so we can create a lock file with flock command
        
        mkdir -p $CACHE_DIR
        LOCK_FILE=$CACHE_DIR/singularity_pull_flock

        # Create an exclusive filelock with flock, this prevents two workflow tasks to pull the same container image at the same time

        flock --exclusive --timeout 15 $LOCK_FILE \
        singularity exec --containall docker://${docker} \
        echo "successfully pulled ${docker}!"

        # Submit the script 
        qsub \
        -W block=true \
        -k oed \
        -v SINGULARITY_CACHEDIR \
        -N ${job_name} \
        -l walltime=${walltime} \
        -q ${queue} \
        -l ncpus=${cpu} \
        -l mem=${memory_gb}gb \
        -o ${cwd}/execution/stdout \
        -e ${cwd}/execution/stderr \
        -- /software/singularity/3.7.2/bin/singularity exec -C --bind ${cwd}:${docker_cwd} docker://${docker} ${job_shell} ${docker_script}
        """
        
        root = "/groups/cgsd/alexandre/cromwell-executions"
        job-id-regex = "(\\d+)"
        kill = "qdel ${job_id}"
        check-alive = "qstat ${job_id}"
      }
    }
  }
}