From 02fb6fe6efa34fb0c0ce4657f56a922a2969d215 Mon Sep 17 00:00:00 2001 From: Heather Ward Date: Mon, 18 Nov 2024 18:04:45 -0500 Subject: [PATCH 1/2] Write source lines rather than str representation of command Parentheses from expressions were being stripped upon materializing the test workflow, resulting in failed submission to Cromwell. e.g. `~{if threads > 1 then "--threads " + (threads - 1) else ""}` was being converted to `~{if threads > 1 then "--threads " + threads - 1 else ""}` This change directly writes out source lines for task commands, rather than writing their string representation, which is for some reason not maintaining parens. --- src/wdlci/cli/submit.py | 7 +++--- src/wdlci/utils/write_workflow.py | 40 ++++++++++++++++++------------- 2 files changed, 28 insertions(+), 19 deletions(-) diff --git a/src/wdlci/cli/submit.py b/src/wdlci/cli/submit.py index 379802a..7ee71a2 100644 --- a/src/wdlci/cli/submit.py +++ b/src/wdlci/cli/submit.py @@ -104,6 +104,7 @@ def submit_handler(kwargs): try: write_workflow( workflow_name, + doc, doc_main_task, output_tests, test_key, @@ -170,9 +171,9 @@ def submit_handler(kwargs): output_test_val_hydrated = HydrateParams.hydrate( source_params, output_test_val, update_key=False ) - output_tests_hydrated[ - output_test_key_hydrated - ] = output_test_val_hydrated + output_tests_hydrated[output_test_key_hydrated] = ( + output_test_val_hydrated + ) workflow_id = submission_state.workflows[test_key]._workflow_id diff --git a/src/wdlci/utils/write_workflow.py b/src/wdlci/utils/write_workflow.py index 380b525..8c99548 100644 --- a/src/wdlci/utils/write_workflow.py +++ b/src/wdlci/utils/write_workflow.py @@ -76,6 +76,7 @@ def _get_output_type(main_task_output_types, output_key): def write_workflow( workflow_name, + main_doc, main_task, output_tests, output_file, @@ -86,6 +87,7 @@ def write_workflow( Write a workflow out to a file Args: workflow_name (str): Name of the test workflow being generated (workflow entrypoint) + main_doc (WDL.Tree.Document): Document from which the main task originated main_task (WDL.Tree.Task): Task to be tested output_tests ({output_name: {"value": output_value, "tasks": ["task0", "task1", "task2"]}}): Array of validated outputs and the test tasks to apply to them. @@ -171,7 +173,7 @@ def write_workflow( except: subprocess.run(["miniwdl", "check", str(test_wdl)]) raise WdlTestCliExitException(f"Invalid test task [{test_wdl}]", 1) - test_tasks[test_task_key] = test_task_doc + test_tasks[test_task_key] = {"task": test_task_doc, "doc": test_doc} f.write( f"{scatter_indent}\tcall {test_task_doc.name} as {test_task_key} {{\n" @@ -196,8 +198,8 @@ def write_workflow( ## Outputs f.write("\n\toutput {\n") - for test_task_key, test_task_doc in test_tasks.items(): - for task_output in test_task_doc.outputs: + for test_task_key, task_info in test_tasks.items(): + for task_output in task_info["task"].outputs: f.write( f"\t\t{task_output.type} {test_task_key}_{task_output.name} = {test_task_key}.{task_output.name}\n" ) @@ -207,12 +209,14 @@ def write_workflow( f.write("}\n") f.write("\n") - _write_task(main_task, output_file) + _write_task(main_doc, main_task, output_file) tasks_written = list() - for test_task_doc in test_tasks.values(): - if test_task_doc.name not in tasks_written: - _write_task(test_task_doc, output_file) - tasks_written.append(test_task_doc.name) + for task_info in test_tasks.values(): + test_doc = task_info["doc"] + test_task = task_info["task"] + if test_task.name not in tasks_written: + _write_task(test_doc, test_task, output_file) + tasks_written.append(test_task.name) # Ensure the workflow is valid try: @@ -223,44 +227,48 @@ def write_workflow( ) -def _write_task(doc_task, output_file): +def _write_task(doc, task, output_file): """ Write a task out to a file Args: - doc_task (WDL.Tree.Task): task to write + doc (WDL.Tree.Document): Document containing task to write + task (WDL.Tree.Task): task to write output_file (str): Path to file to write task to """ with open(output_file, "a") as f: - f.write(f"task {doc_task.name} {{\n") + f.write(f"task {task.name} {{\n") ## Inputs f.write("\tinput " + "{\n") - for task_input in doc_task.inputs: + for task_input in task.inputs: f.write(f"\t\t{task_input}\n") f.write("\t}\n") f.write("\n") ## Post inputs - for post_input in doc_task.postinputs: + for post_input in task.postinputs: f.write(f"\t{post_input}\n") f.write("\n") ## Command f.write("\tcommand <<<\n") - f.write(f"\t\t{doc_task.command}\n") + task_cmd_start = task.command.pos.line + task_cmd_end = task.command.pos.end_line - 1 + for line in doc.source_lines[task_cmd_start:task_cmd_end]: + f.write(f"{line}\n") f.write("\t>>>\n") f.write("\n") ## Outputs f.write("\toutput {\n") - for task_output in doc_task.outputs: + for task_output in task.outputs: f.write(f"\t\t{task_output}\n") f.write("\t}\n") f.write("\n") ## Runtime f.write("\truntime {\n") - for runtime_key, runtime_value in doc_task.runtime.items(): + for runtime_key, runtime_value in task.runtime.items(): f.write(f"\t\t{runtime_key}: {runtime_value}\n") f.write("\t}\n") From d2af995d5b29202d3ed19721175a6b824a16daf0 Mon Sep 17 00:00:00 2001 From: Heather Ward Date: Mon, 18 Nov 2024 18:17:43 -0500 Subject: [PATCH 2/2] Bump version --- README.md | 8 ++++---- action.yml | 10 +++++----- pyproject.toml | 2 +- 3 files changed, 10 insertions(+), 10 deletions(-) diff --git a/README.md b/README.md index 9a3ae53..3ed59ad 100644 --- a/README.md +++ b/README.md @@ -15,7 +15,7 @@ If any step of the action fails, the check will fail; however if some task tests # GitHub action usage ```yaml -- uses: dnastack/wdl-ci@v2.0.0 +- uses: dnastack/wdl-ci@v2.0.1 with: # Configuration file where tests can be found # Default: wdl-ci.config.json @@ -91,7 +91,7 @@ jobs: with: submodules: true - name: wdl-ci - uses: dnastack/wdl-ci@v2.0.0 + uses: dnastack/wdl-ci@v2.0.1 with: wallet-url: ${{ secrets.WALLET_URL }} wallet-client-id: ${{ secrets.WALLET_CLIENT_ID }} @@ -119,7 +119,7 @@ jobs: repository: ${{ github.event.pull_request.head.repo.full_name }} ref: ${{ github.event.pull_request.head.ref }} - name: wdl-ci - uses: dnastack/wdl-ci@v2.0.0 + uses: dnastack/wdl-ci@v2.0.1 with: wallet-url: ${{ secrets.WALLET_URL }} wallet-client-id: ${{ secrets.WALLET_CLIENT_ID }} @@ -149,7 +149,7 @@ jobs: repository: ${{ github.event.pull_request.head.repo.full_name }} ref: ${{ github.event.pull_request.head.ref }} - name: wdl-ci - uses: dnastack/wdl-ci@v2.0.0 + uses: dnastack/wdl-ci@v2.0.1 with: wallet-url: ${{ secrets.WALLET_URL }} wallet-client-id: ${{ secrets.WALLET_CLIENT_ID }} diff --git a/action.yml b/action.yml index e75f7da..5823db2 100644 --- a/action.yml +++ b/action.yml @@ -53,19 +53,19 @@ runs: echo "WDL_CI_CUSTOM_TEST_WDL_DIR"=${{ inputs.wdl-ci-custom-test-wdl-dir }} >> $GITHUB_ENV fi - name: lint - uses: docker://dnastack/wdl-ci:v2.0.0 + uses: docker://dnastack/wdl-ci:v2.0.1 with: args: lint ${{ inputs.suppress-lint-errors && '--suppress-lint-errors' || '' }} - name: detect-changes - uses: docker://dnastack/wdl-ci:v2.0.0 + uses: docker://dnastack/wdl-ci:v2.0.1 with: args: detect-changes - name: submit - uses: docker://dnastack/wdl-ci:v2.0.0 + uses: docker://dnastack/wdl-ci:v2.0.1 with: args: submit - name: monitor - uses: docker://dnastack/wdl-ci:v2.0.0 + uses: docker://dnastack/wdl-ci:v2.0.1 with: args: monitor --update-digests # If a test fails, still update task digests for any tests that succeeded @@ -79,6 +79,6 @@ runs: default_author: github_actions - name: cleanup if: always() - uses: docker://dnastack/wdl-ci:v2.0.0 + uses: docker://dnastack/wdl-ci:v2.0.1 with: args: cleanup diff --git a/pyproject.toml b/pyproject.toml index f3ccc05..5801ff9 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,7 +1,7 @@ [project] name = "wdl-testing-cli" description = "DNAstack WDL testing CLI" -version = "2.0.0" +version = "2.0.1" authors = [ { name = "DNAstack", email = "devs@dnastack.com" } ]