From 97a6815675539ba12a5d52635124db4bdb5c28ac Mon Sep 17 00:00:00 2001 From: Nuwan Goonasekera <2070605+nuwang@users.noreply.github.com> Date: Mon, 7 Mar 2022 20:26:26 +0530 Subject: [PATCH 1/4] Replace container mapper rules with TPV --- galaxy/files/rules/__init__.py | 8 - galaxy/files/rules/k8s_container_mapper.py | 97 ------ .../files/rules/test_k8s_container_mapper.py | 294 ------------------ galaxy/values.yaml | 124 +------- 4 files changed, 12 insertions(+), 511 deletions(-) delete mode 100644 galaxy/files/rules/__init__.py delete mode 100644 galaxy/files/rules/k8s_container_mapper.py delete mode 100644 galaxy/files/rules/test_k8s_container_mapper.py diff --git a/galaxy/files/rules/__init__.py b/galaxy/files/rules/__init__.py deleted file mode 100644 index 620581a8..00000000 --- a/galaxy/files/rules/__init__.py +++ /dev/null @@ -1,8 +0,0 @@ -""" -Switch to the Galaxy application folder (not the Galaxy Helm chart foler), -activate Galaxy venv, and run ``nosetests -v /path/to/galaxy-helm/files/rules/`` -to run all the unit tests. - -To run a single test, use the following syntax: -``nosetests -v --nocapture /path/to/galaxy-helm/files/rules/test_k8s_container_mapper.py:ContainerMapperTest.test_existing_tool_map_small_works`` -""" diff --git a/galaxy/files/rules/k8s_container_mapper.py b/galaxy/files/rules/k8s_container_mapper.py deleted file mode 100644 index c1f63a61..00000000 --- a/galaxy/files/rules/k8s_container_mapper.py +++ /dev/null @@ -1,97 +0,0 @@ -import logging -import os -import re -import yaml - -from galaxy.tools import GALAXY_LIB_TOOLS_UNVERSIONED -from galaxy.jobs import JobDestination - -log = logging.getLogger(__name__) - -CONTAINER_RULE_MAPPER_FILE = os.path.join( - os.path.dirname(__file__), 'container_mapper_rules.yml') - - -def _load_container_mappings(): - if os.path.exists(CONTAINER_RULE_MAPPER_FILE): - with open(CONTAINER_RULE_MAPPER_FILE) as f: - return yaml.safe_load(f) - else: - return {} - - -CONTAINER_RULE_MAP = _load_container_mappings() - - -def _map_resource_set(resource_set_name): - resource_set = CONTAINER_RULE_MAP.get( - 'resources', {}).get('resource_sets', {}).get(resource_set_name) - if resource_set: - mapping = { - 'requests_cpu': resource_set.get( - 'requests', {}).get('cpu'), - 'requests_memory': resource_set.get( - 'requests', {}).get('memory'), - 'limits_cpu': resource_set.get( - 'limits', {}).get('cpu'), - 'limits_memory': resource_set.get( - 'limits', {}).get('memory') - } - # trim empty values - return {k: v for k, v in mapping.items() if v is not None} - else: - raise KeyError( - "Could not find key for resource set: {}".format( - resource_set_name)) - - -def _process_tool_mapping(mapping, params): - override_params = mapping.copy() - # Overwrite with user specified limits - resource_set = override_params.pop('resource_set', None) - if resource_set: - params.update(_map_resource_set(resource_set)) - override_params.pop('tool_ids') - params.update(override_params) - - -def _apply_rule_mappings(tool, params): - if CONTAINER_RULE_MAP: - for mapping in CONTAINER_RULE_MAP.get('mappings', {}): - if isinstance(mapping, str): - mapping = CONTAINER_RULE_MAP.get('mappings', {}).get(mapping) - for mapped_tool_id in mapping.get('tool_ids'): - if re.match(mapped_tool_id, tool.id): - _process_tool_mapping(mapping, params) - return True - return False - - -def _get_default_resource_set_name(): - return CONTAINER_RULE_MAP.get( - 'resources', {}).get('default_resource_set', {}) - - -def k8s_container_mapper(tool, referrer, k8s_runner_id="k8s"): - params = dict(referrer.params) - params['docker_enabled'] = True - # For backwards compatibility: unnest parameters under "container" - cont_map = params.pop("container", {}) - params.update(cont_map) - # 1. First, apply the default resource set (if defined) as job params. - # These will be overridden later by individual tool mappings. - default_resource_set_name = _get_default_resource_set_name() - if default_resource_set_name: - params.update(_map_resource_set(default_resource_set_name)) - # 2. Next, apply resource mappings for individual tools, overwriting the - # defaults. - if not _apply_rule_mappings(tool, params): - # 3. If no explicit rule mapping was defined, and it's a tool that - # requires galaxy_lib, force the default container. Otherwise, - # Galaxy's default container resolution will apply. - if tool.id in GALAXY_LIB_TOOLS_UNVERSIONED or ("CONVERTER_" in tool.id and "CONVERTER_" == tool.id[:10]) or tool.is_datatype_converter: - default_container = params.get('docker_default_container_id') - if default_container: - params['docker_container_id_override'] = default_container - log.debug("[k8s_container_mapper] Dispatching to %s with params %s" % (k8s_runner_id, params)) - return JobDestination(runner=k8s_runner_id, params=params) diff --git a/galaxy/files/rules/test_k8s_container_mapper.py b/galaxy/files/rules/test_k8s_container_mapper.py deleted file mode 100644 index 2f6351a2..00000000 --- a/galaxy/files/rules/test_k8s_container_mapper.py +++ /dev/null @@ -1,294 +0,0 @@ -import sys -import unittest -import yaml - -import k8s_container_mapper - - -class ContainerMapperTest(unittest.TestCase): - - _multiprocess_can_split_ = True - - class Tool: - def __init__(self, tool_id): - self.id = tool_id - - class Referrer: - def __init__(self): - self.params = {} - - def patch_mapper(self, rule_map): - k8s_container_mapper.CONTAINER_RULE_MAP = yaml.safe_load(rule_map) - - def test_existing_tool_map_small_works(self): - RULE_MAP_SMALL_OLD = ''' - mappings: - - tool_ids: - - sort1 - - Grouping1 - container: - docker_container_id_override: test_container - resource_set: small - resources: - resource_sets: - small: - requests: - cpu: 1 - memory: 500m - limits: - cpu: 2 - memory: 1G - default_resource_set: small - ''' - - self.patch_mapper(RULE_MAP_SMALL_OLD) - - destination = k8s_container_mapper.k8s_container_mapper( - ContainerMapperTest.Tool('sort1'), - ContainerMapperTest.Referrer()) - assert(destination.params.get( - 'docker_container_id_override') == 'test_container') - assert (destination.params.get( - 'requests_cpu') == 1) - assert (destination.params.get( - 'requests_memory') == '500m') - assert (destination.params.get( - 'limits_cpu') == 2) - assert (destination.params.get( - 'limits_memory') == '1G') - - RULE_MAP_SMALL = ''' - mappings: - set_small: - tool_ids: - - sort1 - - Grouping1 - docker_container_id_override: test_container - resource_set: small - resources: - resource_sets: - small: - requests: - cpu: 1 - memory: 500m - limits: - cpu: 2 - memory: 1G - default_resource_set: small - ''' - - self.patch_mapper(RULE_MAP_SMALL) - - destination = k8s_container_mapper.k8s_container_mapper( - ContainerMapperTest.Tool('sort1'), - ContainerMapperTest.Referrer()) - assert(destination.params.get( - 'docker_container_id_override') == 'test_container') - assert (destination.params.get( - 'requests_cpu') == 1) - assert (destination.params.get( - 'requests_memory') == '500m') - assert (destination.params.get( - 'limits_cpu') == 2) - assert (destination.params.get( - 'limits_memory') == '1G') - - def test_existing_tool_map_medium_works(self): - RULE_MAP_MEDIUM_OLD = ''' - mappings: - - tool_ids: - - sort1 - - Grouping1 - container: - docker_container_id_override: test_container - resource_set: medium_mem - resources: - resource_sets: - small: - requests: - cpu: 1 - memory: 500m - limits: - cpu: 2 - memory: 1G - medium_mem: - requests: - memory: 1G - limits: - memory: 2G - default_resource_set: small - ''' - - self.patch_mapper(RULE_MAP_MEDIUM_OLD) - - destination = k8s_container_mapper.k8s_container_mapper( - ContainerMapperTest.Tool('Grouping1'), - ContainerMapperTest.Referrer()) - assert(destination.params.get( - 'docker_container_id_override') == 'test_container') - assert (destination.params.get( - 'requests_cpu') == 1) - assert (destination.params.get( - 'requests_memory') == '1G') - assert (destination.params.get( - 'limits_cpu') == 2) - assert (destination.params.get( - 'limits_memory') == '2G') - - RULE_MAP_MEDIUM = ''' - mappings: - tool_ids: - - sort1 - - Grouping1 - docker_container_id_override: test_container - resource_set: medium_mem - resources: - resource_sets: - small: - requests: - cpu: 1 - memory: 500m - limits: - cpu: 2 - memory: 1G - medium_mem: - requests: - memory: 1G - limits: - memory: 2G - default_resource_set: small - ''' - - self.patch_mapper(RULE_MAP_MEDIUM) - - destination = k8s_container_mapper.k8s_container_mapper( - ContainerMapperTest.Tool('Grouping1'), - ContainerMapperTest.Referrer()) - assert(destination.params.get( - 'docker_container_id_override') == 'test_container') - assert (destination.params.get( - 'requests_cpu') == 1) - assert (destination.params.get( - 'requests_memory') == '1G') - assert (destination.params.get( - 'limits_cpu') == 2) - assert (destination.params.get( - 'limits_memory') == '2G') - - def test_nonexisting_tool_resource(self): - RULE_MAP_RESOURCE_NOTEXIST = ''' - mappings: - non_existent: - tool_ids: - - sort1 - - Grouping1 - docker_container_id_override: test_container - resource_set: non_existent - resources: - resource_sets: - small: - requests: - cpu: 1 - memory: 500m - limits: - cpu: 2 - memory: 1G - medium_mem: - requests: - memory: 1G - limits: - memory: 2G - default_resource_set: small - ''' - self.patch_mapper(RULE_MAP_RESOURCE_NOTEXIST) - - if sys.version_info < (3, 1): - assertRegex = self.assertRaisesRegexp - else: - assertRegex = self.assertRaisesRegex - with assertRegex(KeyError, 'non_existent'): - k8s_container_mapper.k8s_container_mapper( - ContainerMapperTest.Tool('Grouping1'), - ContainerMapperTest.Referrer()) - - def test_nonexisting_tool(self): - RULE_MAP_TOOL_NOTEXIST = ''' - mappings: - non_existent: - tool_ids: - - sort1 - - Grouping1 - docker_container_id_override: test_container - resource_set: non_existent - resources: - resource_sets: - small: - requests: - cpu: 1 - memory: 500m - limits: - cpu: 2 - memory: 1G - medium_mem: - requests: - memory: 1G - limits: - memory: 2G - default_resource_set: small - ''' - self.patch_mapper(RULE_MAP_TOOL_NOTEXIST) - - destination = k8s_container_mapper.k8s_container_mapper( - ContainerMapperTest.Tool('UnknownTool'), - ContainerMapperTest.Referrer()) - assert 'docker_container_id_override' not in destination.params - # Should have default resource mappings - assert (destination.params.get( - 'requests_cpu') == 1) - assert (destination.params.get( - 'requests_memory') == '500m') - assert (destination.params.get( - 'limits_cpu') == 2) - assert (destination.params.get( - 'limits_memory') == '1G') - - def test_no_resource_mappings(self): - RULE_MAP_NO_RESOURCES_OLD = ''' - mappings: - - tool_ids: - - sort1 - - Grouping1 - container: - docker_container_id_override: test_container - ''' - self.patch_mapper(RULE_MAP_NO_RESOURCES_OLD) - - destination = k8s_container_mapper.k8s_container_mapper( - ContainerMapperTest.Tool('sort1'), - ContainerMapperTest.Referrer()) - assert(destination.params.get( - 'docker_container_id_override') == 'test_container') - - def test_regex_mappings(self): - RULE_MAP_REGEX = ''' - mappings: - regex: - tool_ids: - - .*data_manager_sam_fasta_index_builder/sam_fasta_index_builder/.* - docker_container_id_override: test_container - ''' - self.patch_mapper(RULE_MAP_REGEX) - - destination = k8s_container_mapper.k8s_container_mapper( - ContainerMapperTest.Tool('toolshed.g2.bx.psu.edu/repos/devteam/data_manager_sam_fasta' - '_index_builder/sam_fasta_index_builder/0.0.3'), - ContainerMapperTest.Referrer()) - assert(destination.params.get( - 'docker_container_id_override') == 'test_container') - destination = k8s_container_mapper.k8s_container_mapper( - ContainerMapperTest.Tool( - 'toolshed.g2.bx.psu.edu/repos/devteam/data_manager_sam_fasta' - '_index_builder/sam_fasta_index_builder/0.0.4'), - ContainerMapperTest.Referrer()) - assert(destination.params.get( - 'docker_container_id_override') == 'test_container') diff --git a/galaxy/values.yaml b/galaxy/values.yaml index ebb5c208..5c303c79 100644 --- a/galaxy/values.yaml +++ b/galaxy/values.yaml @@ -314,18 +314,21 @@ configs: assign: - "db-skip-locked" execution: - default: dynamic_k8s_dispatcher + default: vortex_dispatcher environments: local: runner: local - dynamic_k8s_dispatcher: + vortex_dispatcher: container_monitor: false runner: dynamic type: python - function: k8s_container_mapper + function: map_tool_to_destination + rules_module: vortex.rules docker_enabled: true docker_default_container_id: "{{ .Values.image.repository }}:{{ .Values.image.tag }}" tmp_dir: "true" + vortex_config_files: + {{ .Values.jobs.vortex.rules | toYaml | nindent 8 }} limits: - type: registered_user_concurrent_jobs value: 5 @@ -468,116 +471,13 @@ jobs: priorityClass: enabled: true existingClass: "" + vortex: + rule_files: + - https://github.com/galaxyproject/total-perspective-vortex/raw/main/vortex/tests/fixtures/mapping-rules.yml + # An example local override of remote rules + # - lib/galaxy/jobs/rules/vortex_rules_local.yml rules: - container_mapper_rules.yml: - mappings: - summary_stats: - tool_ids: - - Summary_Statistics1 - docker_container_id_override: cloudve/gsummary:latest - resource_set: small - sam_fasta_dm: - tool_ids: - - toolshed.g2.bx.psu.edu/repos/devteam/data_manager_sam_fasta_index_builder/sam_fasta_index_builder/.* - docker_container_id_override: cloudve/sam-fasta-dm:latest - resource_set: small - bwa_dm: - tool_ids: - - toolshed.g2.bx.psu.edu/repos/devteam/data_manager_bwa_mem_index_builder/bwa_mem_index_builder_data_manager/.* - docker_container_id_override: cloudve/bwa-dm:latest - resource_set: small - prokka: - tool_ids: - - toolshed.g2.bx.psu.edu/repos/crs4/prokka/prokka/1.14.5 - docker_container_id_override: cloudve/prokka:1.14.5 - jbrowse: - tool_ids: - - toolshed.g2.bx.psu.edu/repos/iuc/jbrowse/jbrowse/1.16.5+galaxy6 - docker_container_id_override: cloudve/jbrowse:1.16.5 - lib_galaxy: - tool_ids: - - sort1 - - Grouping1 - docker_container_id_override: "{{ .Values.image.repository }}:{{ .Values.image.tag }}" - resource_set: small - set_medium: - tool_ids: - - toolshed.g2.bx.psu.edu/repos/devteam/bowtie2/bowtie2/.* - - toolshed.g2.bx.psu.edu/repos/iuc/bwameth/bwameth/.* - - toolshed.g2.bx.psu.edu/repos/iuc/featurecounts/featurecounts/.* - - toolshed.g2.bx.psu.edu/repos/iuc/hisat2/hisat2/.* - - toolshed.g2.bx.psu.edu/repos/iuc/valet/valet/.* - - toolshed.g2.bx.psu.edu/repos/iuc/varscan_somatic/varscan_somatic/.* - - toolshed.g2.bx.psu.edu/repos/nilesh/rseqc/rseqc_bam2wig/.* - resource_set: medium - set_large: - tool_ids: - - toolshed.g2.bx.psu.edu/repos/devteam/bwa/bwa_mem/.* - - toolshed.g2.bx.psu.edu/repos/devteam/bwa/bwa/.* - - toolshed.g2.bx.psu.edu/repos/bgruening/deeptools_bam_compare/deeptools_bam_compare/.* - - toolshed.g2.bx.psu.edu/repos/bgruening/deeptools_bam_coverage/deeptools_bam_coverage/.* - - toolshed.g2.bx.psu.edu/repos/bgruening/deeptools_bam_pe_fragmentsize/deeptools_bam_pe_fragmentsize/.* - - toolshed.g2.bx.psu.edu/repos/bgruening/deeptools_bigwig_compare/deeptools_bigwig_compare/.* - - toolshed.g2.bx.psu.edu/repos/bgruening/deeptools_compute_gc_bias/deeptools_compute_gc_bias/.* - - toolshed.g2.bx.psu.edu/repos/bgruening/deeptools_compute_matrix/deeptools_compute_matrix/.* - - toolshed.g2.bx.psu.edu/repos/bgruening/deeptools_correct_gc_bias/deeptools_correct_gc_bias/.* - - toolshed.g2.bx.psu.edu/repos/bgruening/deeptools_multi_bam_summary/deeptools_multi_bam_summary/.* - - toolshed.g2.bx.psu.edu/repos/bgruening/deeptools_multi_bigwig_summary/deeptools_multi_bigwig_summary/.* - - toolshed.g2.bx.psu.edu/repos/devteam/freebayes/freebayes/.* - - toolshed.g2.bx.psu.edu/repos/iuc/rgrnastar/rna_star/.* - - toolshed.g2.bx.psu.edu/repos/iuc/rnaspades/rnaspades/.* - - toolshed.g2.bx.psu.edu/repos/iuc/sra_tools/fasterq_dump/.* - resource_set: large - set_2xlarge: - tool_ids: - - toolshed.g2.bx.psu.edu/repos/iuc/unicycler/unicycler/.* - - toolshed.g2.bx.psu.edu/repos/nml/spades/spades/.* - resource_set: 2xlarge - set_mlarge: - tool_ids: - - toolshed.g2.bx.psu.edu/repos/iuc/minimap2/minimap2/.* - - toolshed.g2.bx.psu.edu/repos/iuc/plink/plink/.* - resource_set: mlarge - resources: - resource_sets: - small: - requests: - cpu: 1 - memory: 2G - limits: - cpu: 2 - memory: 5G - medium: - requests: - cpu: 2 - memory: 4G - limits: - cpu: 4 - memory: 10G - large: - requests: - cpu: 4 - memory: 8G - limits: - cpu: 8 - memory: 16G - 2xlarge: - requests: - cpu: 12 - memory: 20G - limits: - cpu: 12 - memory: 24G - mlarge: - requests: - cpu: 2 - memory: 16G - limits: - cpu: 4 - memory: 20G - default_resource_set: small - k8s_container_mapper.py: | - {{- (.Files.Get "files/rules/k8s_container_mapper.py") }} + vortex_rules_local.yml: extraFileMappings: /galaxy/server/static/welcome.html: From dd72c9a372e2690a8df0da2333560fd00ae9f5c4 Mon Sep 17 00:00:00 2001 From: Nuwan Goonasekera <2070605+nuwang@users.noreply.github.com> Date: Thu, 23 Jun 2022 20:32:16 +0530 Subject: [PATCH 2/4] Match latest TPV release and update local overrides --- galaxy/values.yaml | 43 ++++++++++++++++++++++++++++++++----------- 1 file changed, 32 insertions(+), 11 deletions(-) diff --git a/galaxy/values.yaml b/galaxy/values.yaml index 5c303c79..21380873 100644 --- a/galaxy/values.yaml +++ b/galaxy/values.yaml @@ -314,21 +314,21 @@ configs: assign: - "db-skip-locked" execution: - default: vortex_dispatcher + default: tpv_dispatcher environments: local: runner: local - vortex_dispatcher: + tpv_dispatcher: container_monitor: false runner: dynamic type: python function: map_tool_to_destination - rules_module: vortex.rules + rules_module: tpv.rules docker_enabled: true docker_default_container_id: "{{ .Values.image.repository }}:{{ .Values.image.tag }}" tmp_dir: "true" - vortex_config_files: - {{ .Values.jobs.vortex.rules | toYaml | nindent 8 }} + tpv_config_files: + {{ .Values.jobs.tpv.config_files | toYaml | nindent 8 }} limits: - type: registered_user_concurrent_jobs value: 5 @@ -471,14 +471,35 @@ jobs: priorityClass: enabled: true existingClass: "" - vortex: - rule_files: - - https://github.com/galaxyproject/total-perspective-vortex/raw/main/vortex/tests/fixtures/mapping-rules.yml - # An example local override of remote rules - # - lib/galaxy/jobs/rules/vortex_rules_local.yml + tpv: + config_files: + - https://raw.githubusercontent.com/galaxyproject/tpv-shared-database/main/tools.yml + - lib/galaxy/jobs/rules/vortex_rules_local.yml rules: vortex_rules_local.yml: - + tools: + Summary_Statistics1: + params: + docker_container_id_override: cloudve/gsummary:latest + toolshed.g2.bx.psu.edu/repos/devteam/data_manager_sam_fasta_index_builder/sam_fasta_index_builder/.*: + params: + docker_container_id_override: cloudve/sam-fasta-dm:latest + toolshed.g2.bx.psu.edu/repos/devteam/data_manager_bwa_mem_index_builder/bwa_mem_index_builder_data_manager/.*: + params: + docker_container_id_override: cloudve/bwa-dm:latest + toolshed.g2.bx.psu.edu/repos/crs4/prokka/prokka/1.14.5: + params: + docker_container_id_override: cloudve/prokka:1.14.5 + toolshed.g2.bx.psu.edu/repos/iuc/jbrowse/jbrowse/1.16.5+galaxy6: + params: + docker_container_id_override: cloudve/jbrowse:1.16.5 + sort1: + params: + docker_container_id_override: "{{ .Values.image.repository }}:{{ .Values.image.tag }}" + Grouping1: + params: + docker_container_id_override: "{{ .Values.image.repository }}:{{ .Values.image.tag }}" + extraFileMappings: /galaxy/server/static/welcome.html: useSecret: false From d8ae35e6a18c5ee326bcc0976a1559bd5e8b4f62 Mon Sep 17 00:00:00 2001 From: Nuwan Goonasekera <2070605+nuwang@users.noreply.github.com> Date: Fri, 24 Jun 2022 00:12:25 +0530 Subject: [PATCH 3/4] Quote value to prevent syntax error --- galaxy/values.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/galaxy/values.yaml b/galaxy/values.yaml index 21380873..3dd76db0 100644 --- a/galaxy/values.yaml +++ b/galaxy/values.yaml @@ -328,7 +328,7 @@ configs: docker_default_container_id: "{{ .Values.image.repository }}:{{ .Values.image.tag }}" tmp_dir: "true" tpv_config_files: - {{ .Values.jobs.tpv.config_files | toYaml | nindent 8 }} + "{{ .Values.jobs.tpv.config_files | toYaml | nindent 8 }}" limits: - type: registered_user_concurrent_jobs value: 5 From ce51c15f1935ecbe001b1aaaf0935edb33f698a7 Mon Sep 17 00:00:00 2001 From: Nuwan Goonasekera <2070605+nuwang@users.noreply.github.com> Date: Fri, 24 Jun 2022 00:17:24 +0530 Subject: [PATCH 4/4] Disable traefik to fix github build --- .github/workflows/test.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index d440472e..c4e91acc 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -34,6 +34,7 @@ jobs: uses: jupyterhub/action-k3s-helm@v1 with: k3s-version: v1.19.10+k3s1 # releases: https://github.com/k3s-io/k3s/tags + traefik-enabled: false - name: Verify function of k8s, kubectl, and helm run: | echo "kubeconfig: $KUBECONFIG"