generated from hubverse-org/hubTemplate
-
Notifications
You must be signed in to change notification settings - Fork 12
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Merge pull request #76 from kjsato/main-v3-2
Main v3 2
- Loading branch information
Showing
11 changed files
with
166 additions
and
24 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -25,6 +25,11 @@ jobs: | |
| with: | ||
| install-r: false | ||
| use-public-rspm: true | ||
| - name: Get list of changed files | ||
| id: files | ||
| uses: lots0logs/[email protected] | ||
| with: | ||
| token: ${{ secrets.GITHUB_TOKEN }} | ||
| - name: Install system dependencies | ||
| run: | | ||
| sudo apt-get update | ||
|
|
@@ -61,17 +66,30 @@ jobs: | |
| Rscript -e 'remotes::install_github("Infectious-Disease-Modeling-Hubs/hubVis")' | ||
| - name: Install hubUtils | ||
| run: | | ||
| Rscript -e 'remotes::install_github("Infectious-Disease-Modeling-Hubs/hubUtils")' | ||
| Rscript -e 'remotes::install_github("kjsato/hubUtils@enhancement/v3-utils", upgrade = "never")' | ||
| - name: Install hubEnsembles | ||
| run: | | ||
| Rscript -e 'remotes::install_github("Infectious-Disease-Modeling-Hubs/hubEnsembles")' | ||
| - name: Install hubData | ||
| run: | | ||
| Rscript -e 'remotes::install_github("Infectious-Disease-Modeling-Hubs/hubData")' | ||
| Rscript -e 'remotes::install_github("kjsato/hubData@feature/handle-samples", upgrade = "never")' | ||
| - name: Check if RETRO files are present | ||
| id: check_files | ||
| run: | | ||
| echo 'retro='$(echo '${{ steps.files.outputs.all }}' | grep 'model-output' | grep -c '.*-RETRO.*\.\(csv\|parquet\)') >> $GITHUB_ENV | ||
| echo 'conventional='$(echo '${{ steps.files.outputs.all }}' | grep -c -v '.*-RETRO.*\.\(csv\|parquet\)') >> $GITHUB_ENV | ||
| - name: Run R script | ||
| run: | | ||
| cd ./scripts | ||
| Rscript ensemble.R | ||
| if: env.conventional != '0' | ||
| - name: Run retro_ensemble.R | ||
| run: | | ||
| cd ./scripts | ||
| for file in $(echo '${{ steps.files.outputs.all }}' | grep 'model-output' | grep '.*-RETRO.*\.\(csv\|parquet\)'); do | ||
| Rscript retro_ensemble.R $file | ||
| done | ||
| if: env.retro != '0' | ||
| - name: Commit and push new files to root repository | ||
| run: | | ||
|
|
@@ -90,8 +108,8 @@ jobs: | |
| - name: Checkout and push to another repository | ||
| uses: actions/checkout@v2 | ||
| with: | ||
| #repository: kjsato/rsv-forecast-hub_data | ||
| repository: HopkinsIDD/rsv-forecast-hub_data | ||
| repository: kjsato/rsv-forecast-hub_data | ||
| #repository: HopkinsIDD/rsv-forecast-hub_data | ||
| token: ${{ secrets.KJ3_PATC }} | ||
| path: ./rsv-forecast-hub_data | ||
| fetch-depth: 2 | ||
|
|
||
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -1,12 +1,16 @@ | ||
| { | ||
| "schema_version": "https://raw.githubusercontent.com/Infectious-Disease-Modeling-Hubs/schemas/main/v2.0.0/admin-schema.json", | ||
| "name": "Simple Forecast Hub", | ||
| "maintainer": "Consortium of Infectious Disease Modeling Hubs", | ||
| "schema_version": "https://raw.githubusercontent.com/kjsato/schemas/br-v3.0.0/v3.0.0/admin-schema.json", | ||
| "name": "US RSV Forecast Hub", | ||
| "maintainer": "Johns Hopkins University, Infectious Disease Dynamics Group", | ||
| "contact": { | ||
| "name": "K. Sato", | ||
| "email": "[email protected]" | ||
| }, | ||
| "repository_url": "https://github.com/Infectious-Disease-Modeling-Hubs/hubTemplate", | ||
| "repository": { | ||
| "host": "github", | ||
| "owner": "kjsato", | ||
| "name": "rsv-forecast-hub" | ||
| }, | ||
| "file_format": ["csv", "parquet"], | ||
| "timezone": "US/Eastern" | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,32 @@ | ||
| schema_version: "https://raw.githubusercontent.com/Infectious-Disease-Modeling-Hubs/schemas/main/v2.0.0/admin-schema.json" | ||
| team_name: "Team Sam" | ||
| team_abbr: "teamsam" | ||
| model_name: "Model Ple" | ||
| model_abbr: "modelple" | ||
| model_contributors: [ | ||
| { | ||
| "name": "Smith J", | ||
| "affiliation": "Affiliation", | ||
| "email": "[email protected]" | ||
| }, | ||
| ] | ||
| data_inputs: "Description of Data Inputs" | ||
| methods: "Short description of the model" | ||
| methods_long: "Long description of the model" | ||
|
|
||
| #model_details: [ | ||
| # { | ||
| # "data_inputs": "Description of Data Inputs" | ||
| # }, | ||
| # { | ||
| # "methods": "Short description of the model" | ||
| # }, | ||
| # { | ||
| # "methods_long": "Long description of the model" | ||
| # } | ||
| #] | ||
| license: "MIT" | ||
| model_version: "1.0" | ||
| website_url: "https://url_to_team2_modelb_website.com" | ||
| team_funding: "funding information" | ||
| citation: "Gibson GC , Reich NG , Sheldon D. Real-time mechanistic bayesian forecasts of Covid-19 mortality. medRxiv. 2020. https://doi.org/10.1101/2020.12.22.20248736" |
Binary file modified
BIN
+128 Bytes
(100%)
model-output/CU-RSV_SVIRS/2024-05-12-CU-RSV_SVIRS.parquet
Binary file not shown.
Binary file added
BIN
+36.6 KB
model-output/teamsam-modelple/2023-11-05-teamsam-modelple-RETRO.parquet
Binary file not shown.
Binary file added
BIN
+36.6 KB
model-output/teamsam-modelple/2023-11-12-teamsam-modelple-RETRO.parquet
Binary file not shown.
Binary file added
BIN
+36.6 KB
model-output/teamsam-modelple/2023-11-19-teamsam-modelple-RETRO.parquet
Binary file not shown.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,74 @@ | ||
| ## retro_ensemble.R customized for rsv-forecast-hub | ||
|
|
||
| # Get the file_path from the command line arguments | ||
| args <- commandArgs(trailingOnly = TRUE) | ||
| retro_file_path <- args[1] | ||
|
|
||
| local_path <- paste0(dirname(here::here())) | ||
| dir_path <- file.path(local_path, "rsv-forecast-hub/") | ||
| data_path <- file.path(local_path, "rsv-forecast-hub/") | ||
| print(local_path) | ||
| #dir_path <- local_path | ||
| #data_path <- local_path | ||
| print(dir_path) | ||
|
|
||
| library(hubUtils) | ||
| library(hubData) | ||
| library(hubEnsembles) | ||
| library(dplyr) | ||
| library(purrr) | ||
| library(jsonlite) | ||
|
|
||
| hub_path <- dir_path | ||
| print(hub_path) | ||
| hub_con <- connect_hub(hub_path) | ||
|
|
||
| loc_data <- readr::read_csv(file.path(dir_path, "auxiliary-data/location_census/locations.csv")) | ||
|
|
||
| output_path <- file.path(dir_path, "model-output") | ||
|
|
||
| # Extract date from file name | ||
| curr_origin_date <- as.Date(gsub("^(\\d{4}-\\d{2}-\\d{2}).*", "\\1", basename(retro_file_path))) | ||
|
|
||
| # Get all RETRO files (maybe needed consideration for the case of a combo without RETRO files) | ||
| file_paths <- list.files(output_path, pattern = "-RETRO\\.parquet$|-RETRO\\.csv$", full.names = TRUE, recursive = TRUE) | ||
| file_paths <- file_paths[grepl(curr_origin_date, file_paths)] | ||
| print(file_paths) | ||
|
|
||
| # read the files, and concatenate all the data frames with adding the team name in "model_id" column | ||
| projection_data_all <- file_paths %>% | ||
| map_df(~{ | ||
| # func selection according to the input file format | ||
| read_fun <- ifelse(grepl("\\.parquet$", .x), arrow::read_parquet, readr::read_csv) | ||
|
|
||
| # read data | ||
| data <- read_fun(.x, stringsAsFactors = FALSE) | ||
|
|
||
| # check if 'origin_date' column exists | ||
| if (!"origin_date" %in% names(data)) { | ||
| print(paste("File", .x, "does not contain 'origin_date' column")) | ||
| } | ||
|
|
||
| # append the team name in "model_id" | ||
| data$model_id <- basename(dirname(.x)) | ||
|
|
||
| # return data | ||
| data | ||
| }) | ||
| head(projection_data_all) | ||
|
|
||
| # Prepare data | ||
| projection_data_all <- dplyr::mutate(projection_data_all, | ||
| target_date = as.Date(origin_date) + (horizon * 7) - 1) | ||
| projection_data_all <- as_model_out_tbl(projection_data_all) | ||
|
|
||
| round <- projection_data_all %>% | ||
| dplyr::filter(origin_date == as.Date(curr_origin_date)) %>% | ||
| dplyr::collect() | ||
|
|
||
| # Generate ensemble | ||
| round_ens <- hubEnsembles::simple_ensemble(round) | ||
|
|
||
| # Save ensemble | ||
| dir.create(file.path(dir_path, "model-output", "hub-ensemble"), showWarnings = FALSE, recursive = TRUE) | ||
| arrow::write_parquet(round_ens, file.path(dir_path, "model-output", "hub-ensemble", paste0(curr_origin_date, "-hub-ensemble-RETRO.parquet"))) |