feat: Toil engine for CWL

examples/demo-cwl-project/agc-project.yaml

@@ -0,0 +1,30 @@
+---
+name: CWLDemo
+schemaVersion: 1
+workflows:
+  hello:
+    type:
+      language: cwl
+      version: v1.2
+    sourceURL: workflows/hello/hello.cwl
+  nontrivial:
+    type:
+      language: cwl
+      version: v1.2
+    sourceURL: workflows/nontrivial
+  manyjobs:
+    type:
+      language: cwl
+      version: v1.2
+    sourceURL: workflows/manyjobs
+contexts:
+  myContext:
+    engines:
+      - type: cwl
+        engine: toil
+
+  spotCtx:
+    requestSpotInstances: true
+    engines:
+      - type: cwl
+        engine: toil

examples/demo-cwl-project/workflows/hello/hello.cwl

@@ -0,0 +1,15 @@
+cwlVersion: v1.0
+class: CommandLineTool
+baseCommand: echo
+stdout: output.txt
+inputs:
+  - id: message
+    type: string
+    default: "Hello world!"
+    inputBinding:
+      position: 1
+outputs:
+  - id: output
+    type: File
+    outputBinding:
+      glob: output.txt

examples/demo-cwl-project/workflows/manyjobs/MANIFEST.json

@@ -0,0 +1,7 @@
+{
+    "mainWorkflowURL": "manyjobs.cwl",
+    "inputFileURLs": [
+      "inputs.json"
+    ],
+    "optionsFileURL": "options.json"
+}

examples/demo-cwl-project/workflows/manyjobs/compress-file.cwl

@@ -0,0 +1,18 @@
+#!/usr/bin/env cwl-runner
+cwlVersion: v1.2
+class: CommandLineTool
+baseCommand: ["-9", "-p", "8", "-c"]
+hints:
+  DockerRequirement:
+    dockerPull: bytesco/pigz
+  ResourceRequirement:
+      coresMin: 8
+inputs:
+  input_file:
+    type: File
+    inputBinding:
+      position: 1
+outputs:
+  compressed_file:
+    type: stdout
+stdout: compressed.gz

examples/demo-cwl-project/workflows/manyjobs/concatenate-files.cwl

@@ -0,0 +1,14 @@
+cwlVersion: v1.2
+class: CommandLineTool
+baseCommand: cat
+inputs:
+  files:
+    type: File[]
+    inputBinding:
+      position: 1
+
+outputs:
+  concatenated_file:
+    type: stdout
+
+stdout: concatenated.txt

examples/demo-cwl-project/workflows/manyjobs/inputs.json

@@ -0,0 +1,4 @@
+{
+    "script_file": {"class": "File", "path": "make-a-little-data.js"},
+    "script_argument_count": 1000
+}

examples/demo-cwl-project/workflows/manyjobs/inputs.tiny.json

@@ -0,0 +1,4 @@
+{
+    "script_file": {"class": "File", "path": "make-a-little-data.js"},
+    "script_argument_count": 2
+}

examples/demo-cwl-project/workflows/manyjobs/make-a-little-data.js

@@ -0,0 +1,21 @@
+const crypto = require('crypto')
+if (process.argv.length > 2) {
+    let arg = process.argv[2]
+    console.log("Argument is: " + arg)
+
+    let hashes = [arg]
+
+    for (let i = 10; i >= 0; i--) {
+        console.log(i + " bottles of " + arg + " on the wall...")
+        hasher = crypto.createHash('sha512')
+        for (let h of hashes) {
+            hasher.update(h)
+        }
+        hashes.push(hasher.digest('hex'))
+    }
+
+    console.log("After meditating on the nature of " + arg + ", it turns out to be " + hashes[hashes.length - 1])
+
+} else {
+    console.log("Didn't get an argument")
+}

examples/demo-cwl-project/workflows/manyjobs/make-array.cwl

@@ -0,0 +1,13 @@
+#!/usr/bin/env cwl-runner
+cwlVersion: v1.2
+class: ExpressionTool
+requirements:
+  InlineJavascriptRequirement: {}
+inputs:
+  size:
+    type: int
+outputs:
+  array:
+    type: string[]
+expression: "$({array: function(){var arr = []; for (var i = 0; i < inputs.size; i++) {arr.push('' + i)}; return arr;}()})"
+

examples/demo-cwl-project/workflows/manyjobs/manyjobs.cwl

@@ -0,0 +1,52 @@
+cwlVersion: v1.2
+class: Workflow
+
+requirements:
+  SubworkflowFeatureRequirement: {}
+  ScatterFeatureRequirement: {}
+
+inputs:
+  - id: script_file
+    type: File
+  - id: script_argument_count
+    type: int
+
+steps:
+  make_array:
+    run: make-array.cwl
+    in:
+      size: script_argument_count
+    out: [array]
+
+  scripts:
+    run: run-script.cwl 
+    scatter: arg
+    in:
+      src: script_file
+      arg: make_array/array
+    out: [script_output]
+
+  sort:
+    run: sort-file.cwl 
+    scatter: input_file
+    in:
+      input_file: scripts/script_output
+    out: [sorted_file]
+
+  concat:
+    run: concatenate-files.cwl
+    in:
+      files: sort/sorted_file
+    out:
+      [concatenated_file]
+
+  compress:
+    run: compress-file.cwl 
+    in:
+      input_file: concat/concatenated_file
+    out: [compressed_file]
+
+outputs:
+  - id: output
+    type: File
+    outputSource: compress/compressed_file

examples/demo-cwl-project/workflows/manyjobs/options.json

@@ -0,0 +1 @@
+{}

examples/demo-cwl-project/workflows/manyjobs/run-script.cwl

@@ -0,0 +1,24 @@
+#!/usr/bin/env cwl-runner
+# Modified from the CWL docs
+cwlVersion: v1.2
+class: CommandLineTool
+baseCommand: node
+hints:
+  DockerRequirement:
+    dockerPull: node:slim
+  ResourceRequirement:
+    coresMax: 1
+    ramMin: 2000
+inputs:
+  src:
+    type: File
+    inputBinding:
+      position: 1
+  arg:
+    type: string
+    inputBinding:
+      position: 2
+outputs:
+  script_output:
+    type: stdout
+stdout: output.txt

examples/demo-cwl-project/workflows/manyjobs/sort-file.cwl

@@ -0,0 +1,19 @@
+#!/usr/bin/env cwl-runner
+cwlVersion: v1.2
+class: CommandLineTool
+baseCommand: ["sort"]
+requirements:
+  InlineJavascriptRequirement: {}
+hints:
+  ResourceRequirement:
+    coresMax: 1
+    outdirMin: $(parseInt(Math.ceil(inputs.input_file.size / Math.pow(2, 20))))
+inputs:
+  input_file:
+    type: File
+    inputBinding:
+      position: 1
+outputs:
+  sorted_file:
+    type: stdout
+stdout: sorted.txt

examples/demo-cwl-project/workflows/nontrivial/MANIFEST.json

@@ -0,0 +1,7 @@
+{
+    "mainWorkflowURL": "nontrivial.cwl",
+    "inputFileURLs": [
+      "inputs.json"
+    ],
+    "optionsFileURL": "options.json"
+}

examples/demo-cwl-project/workflows/nontrivial/compress-file.cwl

@@ -0,0 +1,18 @@
+#!/usr/bin/env cwl-runner
+cwlVersion: v1.2
+class: CommandLineTool
+baseCommand: ["-9", "-p", "8", "-c"]
+hints:
+  DockerRequirement:
+    dockerPull: bytesco/pigz
+  ResourceRequirement:
+      coresMin: 8
+inputs:
+  input_file:
+    type: File
+    inputBinding:
+      position: 1
+outputs:
+  compressed_file:
+    type: stdout
+stdout: compressed.gz

examples/demo-cwl-project/workflows/nontrivial/concatenate-files.cwl

@@ -0,0 +1,14 @@
+cwlVersion: v1.2
+class: CommandLineTool
+baseCommand: cat
+inputs:
+  files:
+    type: File[]
+    inputBinding:
+      position: 1
+
+outputs:
+  concatenated_file:
+    type: stdout
+
+stdout: concatenated.txt

examples/demo-cwl-project/workflows/nontrivial/inputs.json

@@ -0,0 +1,7 @@
+{
+    "script_file": {"class": "File", "path": "make-data.js"},
+    "script_arguments": [
+        "A", "B", "C", "D", "E", "F", "G",
+        "cats", "pigs", "dogs", "space weasles", "snacks"
+    ]
+}

examples/demo-cwl-project/workflows/nontrivial/make-data.js

@@ -0,0 +1,21 @@
+const crypto = require('crypto')
+if (process.argv.length > 2) {
+    let arg = process.argv[2]
+    console.log("Argument is: " + arg)
+
+    let hashes = [arg]
+
+    for (let i = 10000; i >= 0; i--) {
+        console.log(i + " bottles of " + arg + " on the wall...")
+        hasher = crypto.createHash('sha512')
+        for (let h of hashes) {
+            hasher.update(h)
+        }
+        hashes.push(hasher.digest('hex'))
+    }
+
+    console.log("After meditating on the nature of " + arg + ", it turns out to be " + hashes[hashes.length - 1])
+
+} else {
+    console.log("Didn't get an argument")
+}

examples/demo-cwl-project/workflows/nontrivial/nontrivial.cwl

@@ -0,0 +1,46 @@
+cwlVersion: v1.2
+class: Workflow
+
+requirements:
+  SubworkflowFeatureRequirement: {}
+  ScatterFeatureRequirement: {}
+
+inputs:
+  - id: script_file
+    type: File
+  - id: script_arguments
+    type: string[]
+
+steps:
+  scripts:
+    run: run-script.cwl 
+    scatter: arg
+    in:
+      src: script_file
+      arg: script_arguments
+    out: [script_output]
+
+  sort:
+    run: sort-file.cwl 
+    scatter: input_file
+    in:
+      input_file: scripts/script_output
+    out: [sorted_file]
+
+  concat:
+    run: concatenate-files.cwl
+    in:
+      files: sort/sorted_file
+    out:
+      [concatenated_file]
+
+  compress:
+    run: compress-file.cwl 
+    in:
+      input_file: concat/concatenated_file
+    out: [compressed_file]
+
+outputs:
+  - id: output
+    type: File
+    outputSource: compress/compressed_file

examples/demo-cwl-project/workflows/nontrivial/options.json

@@ -0,0 +1 @@
+{}

examples/demo-cwl-project/workflows/nontrivial/run-script.cwl

@@ -0,0 +1,24 @@
+#!/usr/bin/env cwl-runner
+# Modified from the CWL docs
+cwlVersion: v1.2
+class: CommandLineTool
+baseCommand: node
+hints:
+  DockerRequirement:
+    dockerPull: node:slim
+  ResourceRequirement:
+    coresMax: 1
+    ramMin: 2000
+inputs:
+  src:
+    type: File
+    inputBinding:
+      position: 1
+  arg:
+    type: string
+    inputBinding:
+      position: 2
+outputs:
+  script_output:
+    type: stdout
+stdout: output.txt