From 72ad7ef8d0193d57ae600e21f9d933d763cfdded Mon Sep 17 00:00:00 2001 From: John Huddleston Date: Fri, 5 Jan 2024 15:27:10 -0800 Subject: [PATCH 1/6] Start adding config/rules to deploy nextflu-private builds --- profiles/nextflu-private.yaml | 9 ++++ profiles/nextflu-private/deploy.smk | 13 ++++++ profiles/nextflu-private/rename.smk | 64 +++++++++++++++++++++++++++++ 3 files changed, 86 insertions(+) create mode 100644 profiles/nextflu-private/deploy.smk create mode 100644 profiles/nextflu-private/rename.smk diff --git a/profiles/nextflu-private.yaml b/profiles/nextflu-private.yaml index fae9cb75..0eb078eb 100644 --- a/profiles/nextflu-private.yaml +++ b/profiles/nextflu-private.yaml @@ -2,6 +2,10 @@ custom_rules: - workflow/snakemake_rules/download_from_s3.smk - profiles/nextflu-private/antigenic_distances.smk - profiles/nextflu-private/report.smk + - profiles/nextflu-private/rename.smk + +# URL for auto-deploying builds +deploy_url: https://nextstrain.org/groups/nextflu-private/ fasta_fields: - strain @@ -46,6 +50,7 @@ recency: builds: h1n1pdm_2y_titers: + auspice_name: "flu_seasonal_{build_date}_h1n1pdm_2y_titers_{segment}" lineage: h1n1pdm reference: "config/h1n1pdm/{segment}/reference.fasta" annotation: "config/h1n1pdm/{segment}/genemap.gff" @@ -94,6 +99,8 @@ builds: # enabling titer models to be fit to egg-passaged data. filters: --query "(is_titer_strain == True)" --min-date {min_date} --exclude {exclude} h3n2_2y_titers: + # flu/seasonal/2023-10-26/h3n2/2y/titers/ha + auspice_name: "flu_seasonal_{build_date}_h3n2_2y_titers_{segment}" lineage: "h3n2" reference: "config/h3n2/{segment}/reference.fasta" annotation: "config/h3n2/{segment}/genemap.gff" @@ -139,6 +146,7 @@ builds: title: "Cell-passaged HI titers from pooled human sera" subsamples: *titers-subsampling-scheme h3n2_2y: + auspice_name: "flu_seasonal_{build_date}_h3n2_2y_{segment}" lineage: "h3n2" reference: "config/h3n2/{segment}/reference.fasta" annotation: "config/h3n2/{segment}/genemap.gff" @@ -176,6 +184,7 @@ builds: references: filters: --query "(is_reference == True)" --min-date {reference_min_date} --exclude {exclude} vic_2y_titers: + auspice_name: "flu_seasonal_{build_date}_vic_2y_titers_{segment}" lineage: vic reference: "config/vic/{segment}/reference.fasta" annotation: "config/vic/{segment}/genemap.gff" diff --git a/profiles/nextflu-private/deploy.smk b/profiles/nextflu-private/deploy.smk new file mode 100644 index 00000000..7457f303 --- /dev/null +++ b/profiles/nextflu-private/deploy.smk @@ -0,0 +1,13 @@ +""" +This part of the workflow handles automatic deployments of nextflu-private builds. +Depends on the `all_private` rule from rename.smk +""" + +rule deploy_all: + input: rules.all_private.input + params: + deploy_url = config["deploy_url"] + shell: + """ + nextstrain remote upload {params.deploy_url} {input} + """ diff --git a/profiles/nextflu-private/rename.smk b/profiles/nextflu-private/rename.smk new file mode 100644 index 00000000..28f487c6 --- /dev/null +++ b/profiles/nextflu-private/rename.smk @@ -0,0 +1,64 @@ +BUILD_DATE = datetime.date.today().strftime("%Y-%m-%d") + +all_private_builds = [ + "auspice_renamed/" + build.get("auspice_name", f"{build_name}_{segment}").format(build_date=BUILD_DATE, segment=segment) + suffix + ".json" + for build_name, build in config["builds"].items() + for segment in config["segments"] + for suffix in ["", "_root-sequence", "_tip-frequencies"] +] + +rule all_private: + input: + all_private_builds + +def _get_file_by_auspice_name(wildcards): + """Find the original Auspice JSON file that needs to be renamed. + """ + for build_name, build_params in config["builds"].items(): + for segment in config["segments"]: + if build_params.get("auspice_name", f"{build_name}_{{segment}}").format(build_date=BUILD_DATE, segment=segment) == wildcards.auspice_name: + return f"auspice/{build_name}_{segment}.json" + + return "" + +rule rename_auspice_main: + input: + _get_file_by_auspice_name, + output: + "auspice_renamed/{auspice_name}.json", + shell: + """ + ln {input} {output} + """ + +rule rename_auspice_root_sequence: + input: + lambda wildcards: _get_file_by_auspice_name(wildcards).replace(".json", "_root-sequence.json"), + output: + "auspice_renamed/{auspice_name}_root-sequence.json", + shell: + """ + ln {input} {output} + """ + +rule rename_auspice_tip_frequencies: + input: + lambda wildcards: _get_file_by_auspice_name(wildcards).replace(".json", "_tip-frequencies.json"), + output: + "auspice_renamed/{auspice_name}_tip-frequencies.json", + shell: + """ + ln {input} {output} + """ + +# auspice_renamed/flu_seasonal_2024-01-05_h3n2_2y_titers_ha.json +# {auspice_name}_{forecast_model}_forecast-tip-frequencies.json +# rule rename_auspice_tip_frequencies: +# input: +# lambda wildcards: _get_file_by_auspice_name(wildcards).replace(".json", "_tip-frequencies.json"), +# output: +# "auspice_renamed/{auspice_name}_{forecast_model}_forecast-tip-frequencies.json", +# shell: +# """ +# ln {input} {output} +# """ From 53adc51de6ddb194c4c2cabe43661aeef5b8d7d6 Mon Sep 17 00:00:00 2001 From: John Huddleston Date: Mon, 8 Jan 2024 14:39:00 -0800 Subject: [PATCH 2/6] Setup CLI auth for nextflu-private deploy Explicitly exports the Nextstrain username and password for the nextflu-private group to the AWS Batch environment from the GitHub Actions environment and runs nextstrain login prior to running the remote upload command to enable the upload command from the workflow. --- .github/workflows/run-nextflu-private-builds.yaml | 4 +++- profiles/nextflu-private.yaml | 1 + profiles/nextflu-private/deploy.smk | 1 + 3 files changed, 5 insertions(+), 1 deletion(-) diff --git a/.github/workflows/run-nextflu-private-builds.yaml b/.github/workflows/run-nextflu-private-builds.yaml index 772e68d4..07be8982 100644 --- a/.github/workflows/run-nextflu-private-builds.yaml +++ b/.github/workflows/run-nextflu-private-builds.yaml @@ -25,8 +25,10 @@ jobs: --memory 72gib \ --env AWS_ACCESS_KEY_ID \ --env AWS_SECRET_ACCESS_KEY \ + --env NEXTSTRAIN_USERNAME \ + --env NEXTSTRAIN_PASSWORD \ . \ - all \ + deploy_all \ all_counts_of_recent_tips_by_clade \ -p \ --configfile profiles/nextflu-private.yaml diff --git a/profiles/nextflu-private.yaml b/profiles/nextflu-private.yaml index 0eb078eb..143aff31 100644 --- a/profiles/nextflu-private.yaml +++ b/profiles/nextflu-private.yaml @@ -3,6 +3,7 @@ custom_rules: - profiles/nextflu-private/antigenic_distances.smk - profiles/nextflu-private/report.smk - profiles/nextflu-private/rename.smk + - profiles/nextflu-private/deploy.smk # URL for auto-deploying builds deploy_url: https://nextstrain.org/groups/nextflu-private/ diff --git a/profiles/nextflu-private/deploy.smk b/profiles/nextflu-private/deploy.smk index 7457f303..35b832a1 100644 --- a/profiles/nextflu-private/deploy.smk +++ b/profiles/nextflu-private/deploy.smk @@ -9,5 +9,6 @@ rule deploy_all: deploy_url = config["deploy_url"] shell: """ + nextstrain login; nextstrain remote upload {params.deploy_url} {input} """ From bc1e2c9f4fdc40e97def648eff05339ee68ca7d8 Mon Sep 17 00:00:00 2001 From: John Huddleston Date: Mon, 8 Jan 2024 14:47:38 -0800 Subject: [PATCH 3/6] Do not prompt for credentials on login Credentials should be available in the environment. --- profiles/nextflu-private/deploy.smk | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/profiles/nextflu-private/deploy.smk b/profiles/nextflu-private/deploy.smk index 35b832a1..13f51664 100644 --- a/profiles/nextflu-private/deploy.smk +++ b/profiles/nextflu-private/deploy.smk @@ -9,6 +9,6 @@ rule deploy_all: deploy_url = config["deploy_url"] shell: """ - nextstrain login; + nextstrain login --no-prompt; nextstrain remote upload {params.deploy_url} {input} """ From 31871c5afeedff6c568688d61565dce323ce3362 Mon Sep 17 00:00:00 2001 From: John Huddleston Date: Thu, 11 Jan 2024 16:40:11 -0800 Subject: [PATCH 4/6] Simplify renaming logic for nextflu-private deploy Simplifies the logic to rename Auspice JSONs for nextflu-private builds to match what we were already doing manually and collects the "rename" and "deploy" rules into a single Snakemake file. --- Snakefile | 8 ++-- profiles/nextflu-private.yaml | 6 --- profiles/nextflu-private/deploy.smk | 21 ++++++++-- profiles/nextflu-private/rename.smk | 64 ----------------------------- 4 files changed, 22 insertions(+), 77 deletions(-) delete mode 100644 profiles/nextflu-private/rename.smk diff --git a/Snakefile b/Snakefile index 2c1c69f1..5ee0aec9 100644 --- a/Snakefile +++ b/Snakefile @@ -37,10 +37,6 @@ include: "workflow/snakemake_rules/fitness.smk" include: "workflow/snakemake_rules/report.smk" -if "custom_rules" in config: - for rule_file in config["custom_rules"]: - include: rule_file - def _get_build_outputs(): outputs = [] for build_name, build_params in config["builds"].items(): @@ -60,6 +56,10 @@ def _get_build_outputs(): return outputs +if "custom_rules" in config: + for rule_file in config["custom_rules"]: + include: rule_file + rule all: input: _get_build_outputs() diff --git a/profiles/nextflu-private.yaml b/profiles/nextflu-private.yaml index 143aff31..001e1df2 100644 --- a/profiles/nextflu-private.yaml +++ b/profiles/nextflu-private.yaml @@ -2,7 +2,6 @@ custom_rules: - workflow/snakemake_rules/download_from_s3.smk - profiles/nextflu-private/antigenic_distances.smk - profiles/nextflu-private/report.smk - - profiles/nextflu-private/rename.smk - profiles/nextflu-private/deploy.smk # URL for auto-deploying builds @@ -51,7 +50,6 @@ recency: builds: h1n1pdm_2y_titers: - auspice_name: "flu_seasonal_{build_date}_h1n1pdm_2y_titers_{segment}" lineage: h1n1pdm reference: "config/h1n1pdm/{segment}/reference.fasta" annotation: "config/h1n1pdm/{segment}/genemap.gff" @@ -100,8 +98,6 @@ builds: # enabling titer models to be fit to egg-passaged data. filters: --query "(is_titer_strain == True)" --min-date {min_date} --exclude {exclude} h3n2_2y_titers: - # flu/seasonal/2023-10-26/h3n2/2y/titers/ha - auspice_name: "flu_seasonal_{build_date}_h3n2_2y_titers_{segment}" lineage: "h3n2" reference: "config/h3n2/{segment}/reference.fasta" annotation: "config/h3n2/{segment}/genemap.gff" @@ -147,7 +143,6 @@ builds: title: "Cell-passaged HI titers from pooled human sera" subsamples: *titers-subsampling-scheme h3n2_2y: - auspice_name: "flu_seasonal_{build_date}_h3n2_2y_{segment}" lineage: "h3n2" reference: "config/h3n2/{segment}/reference.fasta" annotation: "config/h3n2/{segment}/genemap.gff" @@ -185,7 +180,6 @@ builds: references: filters: --query "(is_reference == True)" --min-date {reference_min_date} --exclude {exclude} vic_2y_titers: - auspice_name: "flu_seasonal_{build_date}_vic_2y_titers_{segment}" lineage: vic reference: "config/vic/{segment}/reference.fasta" annotation: "config/vic/{segment}/genemap.gff" diff --git a/profiles/nextflu-private/deploy.smk b/profiles/nextflu-private/deploy.smk index 13f51664..9f3ff75b 100644 --- a/profiles/nextflu-private/deploy.smk +++ b/profiles/nextflu-private/deploy.smk @@ -1,14 +1,29 @@ """ This part of the workflow handles automatic deployments of nextflu-private builds. -Depends on the `all_private` rule from rename.smk """ +rule all_private: + input: + jsons=_get_build_outputs(), + output: + json_dir=directory("auspice_renamed"), + params: + build_date=datetime.date.today().strftime("%Y-%m-%d"), + shell: + """ + for file in {input.jsons} + do + ln ${{file}} {output.json_dir}/"flu_seasonal_{params.build_date}_`basename ${{file}}`" + done + """ + rule deploy_all: - input: rules.all_private.input + input: + json_dir=directory("auspice_renamed"), params: deploy_url = config["deploy_url"] shell: """ nextstrain login --no-prompt; - nextstrain remote upload {params.deploy_url} {input} + nextstrain remote upload {params.deploy_url} {input.json_dir}/*.json """ diff --git a/profiles/nextflu-private/rename.smk b/profiles/nextflu-private/rename.smk deleted file mode 100644 index 28f487c6..00000000 --- a/profiles/nextflu-private/rename.smk +++ /dev/null @@ -1,64 +0,0 @@ -BUILD_DATE = datetime.date.today().strftime("%Y-%m-%d") - -all_private_builds = [ - "auspice_renamed/" + build.get("auspice_name", f"{build_name}_{segment}").format(build_date=BUILD_DATE, segment=segment) + suffix + ".json" - for build_name, build in config["builds"].items() - for segment in config["segments"] - for suffix in ["", "_root-sequence", "_tip-frequencies"] -] - -rule all_private: - input: - all_private_builds - -def _get_file_by_auspice_name(wildcards): - """Find the original Auspice JSON file that needs to be renamed. - """ - for build_name, build_params in config["builds"].items(): - for segment in config["segments"]: - if build_params.get("auspice_name", f"{build_name}_{{segment}}").format(build_date=BUILD_DATE, segment=segment) == wildcards.auspice_name: - return f"auspice/{build_name}_{segment}.json" - - return "" - -rule rename_auspice_main: - input: - _get_file_by_auspice_name, - output: - "auspice_renamed/{auspice_name}.json", - shell: - """ - ln {input} {output} - """ - -rule rename_auspice_root_sequence: - input: - lambda wildcards: _get_file_by_auspice_name(wildcards).replace(".json", "_root-sequence.json"), - output: - "auspice_renamed/{auspice_name}_root-sequence.json", - shell: - """ - ln {input} {output} - """ - -rule rename_auspice_tip_frequencies: - input: - lambda wildcards: _get_file_by_auspice_name(wildcards).replace(".json", "_tip-frequencies.json"), - output: - "auspice_renamed/{auspice_name}_tip-frequencies.json", - shell: - """ - ln {input} {output} - """ - -# auspice_renamed/flu_seasonal_2024-01-05_h3n2_2y_titers_ha.json -# {auspice_name}_{forecast_model}_forecast-tip-frequencies.json -# rule rename_auspice_tip_frequencies: -# input: -# lambda wildcards: _get_file_by_auspice_name(wildcards).replace(".json", "_tip-frequencies.json"), -# output: -# "auspice_renamed/{auspice_name}_{forecast_model}_forecast-tip-frequencies.json", -# shell: -# """ -# ln {input} {output} -# """ From 7ab89bb2b102dcba55d7dd44f2724ef4d3c1c463 Mon Sep 17 00:00:00 2001 From: John Huddleston Date: Fri, 12 Jan 2024 09:39:45 -0800 Subject: [PATCH 5/6] Create directory for renamed Auspice JSONs Snakemake does not automatically create output directories that have been flagged with the `directory` function, so we need to manually create the output directory before we can link renamed Auspice JSONs there. --- profiles/nextflu-private/deploy.smk | 1 + 1 file changed, 1 insertion(+) diff --git a/profiles/nextflu-private/deploy.smk b/profiles/nextflu-private/deploy.smk index 9f3ff75b..ee2fbc0a 100644 --- a/profiles/nextflu-private/deploy.smk +++ b/profiles/nextflu-private/deploy.smk @@ -11,6 +11,7 @@ rule all_private: build_date=datetime.date.today().strftime("%Y-%m-%d"), shell: """ + mkdir -p {output.json_dir}; for file in {input.jsons} do ln ${{file}} {output.json_dir}/"flu_seasonal_{params.build_date}_`basename ${{file}}`" From 66ffc66c60e5a0df7fdd15eea434eff0a58389f0 Mon Sep 17 00:00:00 2001 From: John Huddleston Date: Fri, 12 Jan 2024 09:46:33 -0800 Subject: [PATCH 6/6] Update nextflu-private instructions Remove manual JSON renaming and deployment instructions from the nextflu-private instructions. --- profiles/nextflu-private/README.md | 26 ++++---------------------- 1 file changed, 4 insertions(+), 22 deletions(-) diff --git a/profiles/nextflu-private/README.md b/profiles/nextflu-private/README.md index 9b06cc66..2772c948 100644 --- a/profiles/nextflu-private/README.md +++ b/profiles/nextflu-private/README.md @@ -1,28 +1,10 @@ # Monthly reports on seasonal influenza evolution -## Build trees +## Build and deploy trees -[Run the nextflu-private build with the GitHub Action](https://github.com/nextstrain/seasonal-flu/actions/workflows/run-nextflu-private-builds.yaml). -Use the build summary for the job to track the progress of the builds on AWS Batch and download the final builds. -If trees look reasonable, rename them to match the current date (or the date corresponding to when the data were updated). - -``` bash -export DATE=2022-10-03 -cd auspice/ - -for file in {h1n1pdm,h3n2,vic}* -do - mv ${file} "flu_seasonal_${DATE}_${file}" -done -``` - -Upload the files to the Nextstrain group. - -``` bash -nextstrain remote upload \ - https://nextstrain.org/groups/nextflu-private/ \ - *.json -``` +[Run the nextflu-private builds with the GitHub Action](https://github.com/nextstrain/seasonal-flu/actions/workflows/run-nextflu-private-builds.yaml). +The workflow automatically deploys dated Auspice JSONs to [the nextflu-private group](https://nextstrain.org/groups/nextflu-private/) (e.g., https://nextstrain.org/groups/nextflu-private/flu/seasonal/2024-01-08/h3n2/2y/ha). +View the GitHub Action summary for details about how to download the build artifacts from AWS Batch. ## Prepare a report