Merge pull request #27 from DNAstack/dont-purge-namespace

Dont purge all custom workflows from namespace

README.md

@@ -15,7 +15,7 @@ If any step of the action fails, the check will fail; however if some task tests
 # GitHub action usage
 
 ```yaml
-- uses: dnastack/[email protected].4
+- uses: dnastack/[email protected].6
   with:
     # Configuration file where tests can be found
     # Default: wdl-ci.config.json
@@ -86,7 +86,7 @@ jobs:
         with:
           submodules: true
       - name: wdl-ci
-        uses: dnastack/[email protected].5
+        uses: dnastack/[email protected].6
         with:
           wallet-url: ${{ secrets.WALLET_URL }}
           wallet-client-id: ${{ secrets.WALLET_CLIENT_ID }}
@@ -114,7 +114,7 @@ jobs:
           repository: ${{ github.event.pull_request.head.repo.full_name }}
           ref: ${{ github.event.pull_request.head.ref }}
       - name: wdl-ci
-        uses: dnastack/[email protected].5
+        uses: dnastack/[email protected].6
         with:
           wallet-url: ${{ secrets.WALLET_URL }}
           wallet-client-id: ${{ secrets.WALLET_CLIENT_ID }}
@@ -144,7 +144,7 @@ jobs:
           repository: ${{ github.event.pull_request.head.repo.full_name }}
           ref: ${{ github.event.pull_request.head.ref }}
       - name: wdl-ci
-        uses: dnastack/[email protected].5
+        uses: dnastack/[email protected].6
         with:
           wallet-url: ${{ secrets.WALLET_URL }}
           wallet-client-id: ${{ secrets.WALLET_CLIENT_ID }}

action.yml

@@ -50,19 +50,19 @@ runs:
           echo "WDL_CI_CUSTOM_TEST_WDL_DIR"=${{ inputs.wdl_ci_custom_test_wdl_dir }} >> $GITHUB_ENV
         fi
     - name: lint
-      uses: docker://dnastack/wdl-ci:v0.1.5
+      uses: docker://dnastack/wdl-ci:v0.1.6
       with:
         args: lint
     - name: detect-changes
-      uses: docker://dnastack/wdl-ci:v0.1.5
+      uses: docker://dnastack/wdl-ci:v0.1.6
       with:
         args: detect-changes
     - name: submit
-      uses: docker://dnastack/wdl-ci:v0.1.5
+      uses: docker://dnastack/wdl-ci:v0.1.6
       with:
         args: submit
     - name: monitor
-      uses: docker://dnastack/wdl-ci:v0.1.5
+      uses: docker://dnastack/wdl-ci:v0.1.6
       with:
         args: monitor --update-digests
       # If a test fails, still update task digests for any tests that succeeded
@@ -76,6 +76,6 @@ runs:
         default_author: github_actions
     - name: cleanup
       if: always()
-      uses: docker://dnastack/wdl-ci:v0.1.5
+      uses: docker://dnastack/wdl-ci:v0.1.6
       with:
         args: cleanup

pyproject.toml

@@ -1,7 +1,7 @@
 [project]
 name = "wdl-testing-cli"
 description = "DNAstack WDL testing CLI"
-version = "v0.1.5"
+version = "v0.1.6"
 authors = [
     { name = "DNAstack", email = "[email protected]" }
 ]

src/wdlci/cli/cleanup.py

@@ -1,3 +1,4 @@
+import jsonpickle
 import os
 import sys
 from wdlci.auth.refresh_token_auth import RefreshTokenAuth
@@ -40,9 +41,9 @@ def cleanup_handler(kwargs):
         )
         workflow_service_client = WorkflowServiceClient(workflow_service_auth)
 
-        # purge all custom workflows in the namespace
-        # TODO maybe add option here to purge all custom workflows, or only those registered in state file
-        workflow_service_client.purge_custom_workflows()
+        submission_state = jsonpickle.decode(open(SUBMISSION_JSON, "r").read())
+        for workflow in submission_state.workflows.values():
+            workflow_service_client.delete_custom_workflow(workflow)
 
         print("Cleanup complete. Custom workflows purged from namespace")
 

src/wdlci/cli/submit.py

@@ -116,12 +116,17 @@ def submit_handler(kwargs):
                             )
                             sys.exit(e.exit_code)
 
-                        workflow_id = workflow_service_client.register_workflow(
+                        (
+                            workflow_id,
+                            workflow_etag,
+                        ) = workflow_service_client.register_workflow(
                             test_key,
                             workflow_config,
                             transient=True,
                         )
-                        submission_state.add_workflow(test_key, workflow_id)
+                        submission_state.add_workflow(
+                            test_key, workflow_id, workflow_etag
+                        )
 
                         tasks_to_test[test_key] = {
                             "task": task,

src/wdlci/model/submission_state.py

@@ -7,9 +7,9 @@ def __init__(self):
         self.engines = {}
         self.workflow_runs = []
 
-    def add_workflow(self, workflow_key, workflow_id):
+    def add_workflow(self, workflow_key, workflow_id, workflow_etag):
         self.workflows[workflow_key] = SubmissionStateWorkflow(
-            workflow_key, workflow_id
+            workflow_key, workflow_id, workflow_etag
         )
 
     def add_engine(self, engine_id, engine_json):
@@ -31,9 +31,10 @@ def submit_all(self):
 
 
 class SubmissionStateWorkflow(object):
-    def __init__(self, workflow_key, workflow_id):
+    def __init__(self, workflow_key, workflow_id, workflow_etag):
         self._workflow_key = workflow_key
         self._workflow_id = workflow_id
+        self._workflow_etag = workflow_etag
 
     @property
     def workflow_key(self):
@@ -51,6 +52,14 @@ def workflow_id(self):
     def workflow_id(self, workflow_id):
         self._workflow_id = workflow_id
 
+    @property
+    def workflow_etag(self):
+        return self._workflow_etag
+
+    @workflow_etag.setter
+    def workflow_etag(self, workflow_etag):
+        return self._workflow_etag
+
 
 class SubmissionStateWorkflowRun(object):
     STATUS_UNSUBMITTED = "UNSUBMITTED"

src/wdlci/workbench/workflow_service_client.py

@@ -43,8 +43,31 @@ def register_workflow(self, workflow_key, workflow_config, transient=False):
                 f"Could not register workflow [{workflow_key}] on Workbench", 1
             )
 
-        return response.json()["id"]
+        workflow_id = response.json()["id"]
+        workflow_etag = response.headers.get("Etag").strip('"')
+        return workflow_id, workflow_etag
 
+    # Delete individual custom workflows
+    def delete_custom_workflow(self, workflow):
+        env = Config.instance().env
+        base_url, namespace = (
+            env.workbench_workflow_service_url,
+            env.workbench_namespace,
+        )
+        url = f"{base_url}/{namespace}/workflows/{workflow.workflow_id}"
+        headers = {
+            "Authorization": "Bearer " + self.workflow_service_auth.access_token,
+            "If-Match": workflow.workflow_etag,
+        }
+
+        response = requests.delete(url, headers=headers)
+        if response.status_code != 204:
+            raise WdlTestCliExitException(
+                f"Error when deleting custom workflow {workflow.workflow_key} ({workflow.workflow_id})",
+                1,
+            )
+
+    # Purge all custom workflows
     def purge_custom_workflows(self):
         env = Config.instance().env
         base_url, namespace = (