From 781e6643dc4dc88006bb074cb46283d6200721f8 Mon Sep 17 00:00:00 2001 From: suecharo Date: Tue, 30 Jan 2024 13:31:44 +0900 Subject: [PATCH] Add feature upload-zenodo subcommand --- README.md | 49 +++++++++++++++++++++++++++++++++++------ src/args.rs | 43 ++++++++++++++++++++++++++++++++++++ src/main.rs | 23 +++++++++++++++++++ src/sub_cmd.rs | 41 ++++++++++++++++++++++++++++++++++ src/sub_cmd/validate.rs | 16 +++++--------- src/zenodo.rs | 4 ++-- 6 files changed, 157 insertions(+), 19 deletions(-) diff --git a/README.md b/README.md index 0d36c65..753abd0 100644 --- a/README.md +++ b/README.md @@ -51,7 +51,7 @@ This section describes some subcommands. ```bash $ yevis --help -yevis 0.4.0 +yevis 0.5.8 DDBJ(Bioinformatics and DDBJ Center) CLI tool that supports building a Yevis workflow registry with automated quality control @@ -69,6 +69,7 @@ SUBCOMMANDS: (`CI=true`)) pull-request Create a pull request based on the Yevis metadata files (after validation and testing) test Test workflow based on the Yevis metadata files + upload-zenodo Upload dataset to Zenodo validate Validate schema and contents of the Yevis metadata file ``` @@ -78,7 +79,7 @@ Generate a workflow metadata file template from a primary workflow file URL. ```bash $ yevis make-template --help -yevis-make-template 0.4.0 +yevis-make-template 0.5.8 Generate a template file for the Yevis metadata file USAGE: @@ -109,7 +110,7 @@ Validate schema and contents of the workflow metadata file. ```bash $ yevis validate --help -yevis-validate 0.4.0 +yevis-validate 0.5.8 Validate schema and contents of the Yevis metadata file USAGE: @@ -159,7 +160,7 @@ Test workflow using [GA4GH WES](https://www.ga4gh.org/news/ga4gh-wes-api-enables ```bash $ yevis test --help -yevis-test 0.4.0 +yevis-test 0.5.8 Test workflow based on the Yevis metadata files USAGE: @@ -232,7 +233,7 @@ Create a pull request after validation and testing. ```bash $ yevis pull-request --help -yevis-pull-request 0.4.0 +yevis-pull-request 0.5.8 Create a pull request based on the Yevis metadata files (after validation and testing) USAGE: @@ -269,7 +270,7 @@ Upload files to Zenodo, generate TRS responses and deploy them on GitHub Pages. ```bash $ yevis publish --help -yevis-publish 0.4.0 +yevis-publish 0.5.8 Generate TRS responses and host them on GitHub Pages. (Basically used in the CI environment (`CI=true`)) USAGE: @@ -309,6 +310,40 @@ Note that the following four options: See the GitHub Actions section for more details. +### upload-zenodo + +Upload files in the Yevis metadata to Zenodo and replace the metadata file with the Zenodo URL. + +```bash +$ yevis upload-zenodo --help +yevis-upload-zenodo 0.5.8 +Upload dataset to Zenodo + +USAGE: + yevis upload-zenodo [FLAGS] [OPTIONS] --repository [metadata-location] + +FLAGS: + -h, --help Prints help information + -V, --version Prints version information + -v, --verbose Verbose mode + +OPTIONS: + --gh-token GitHub Personal Access Token + -o, --output Path to the output file [default: yevis-metadata-uploaded.yml] + -r, --repository GitHub repository that publishes TRS responses (format: /) + --zenodo-community Community set in Zenodo deposition + --zenodo-host + Zenodo host. Uses zenodo.org by default and sandbox.zenodo.org for dev-mode + + --zenodo-token + Zenodo Personal Access Token. You can generate it at + https://zenodo.org/account/settings/applications/tokens/new/ + +ARGS: + Location of the Yevis metadata file (local file path or remote URL) [default: yevis- + metadata.yml] +``` + #### Generated TRS Responses Please note, as raised in the issue by @kinow, that the TRS responses generated by Yevis may not be fully compliant with the [TRS API](https://editor.swagger.io/?url=https://raw.githubusercontent.com/ga4gh/tool-registry-schemas/develop/openapi/openapi.yaml). @@ -393,7 +428,7 @@ yevis 0.4.0 ### Build binary -Recommendation, build the binary using `musl``: +Recommendation, build the binary using `musl`: ```bash $ docker run --rm -it -v $PWD:/home/rust/src ekidd/rust-musl-builder cargo build --release diff --git a/src/args.rs b/src/args.rs index 46ad6dc..c9902bf 100644 --- a/src/args.rs +++ b/src/args.rs @@ -160,6 +160,47 @@ pub enum Args { #[structopt(short, long)] verbose: bool, }, + + #[structopt(setting(clap::AppSettings::ColoredHelp))] + /// Upload dataset to Zenodo. + UploadZenodo { + /// Location of the Yevis metadata file (local file path or remote URL). + #[structopt(default_value = "yevis-metadata.yml")] + metadata_location: String, + + /// GitHub Personal Access Token. + #[structopt(long = "gh-token")] + github_token: Option, + + /// GitHub repository that publishes TRS responses (format: /). + #[structopt(short, long)] + repository: String, + + /// Zenodo Personal Access Token. You can generate it at https://zenodo.org/account/settings/applications/tokens/new/. + #[structopt(long = "zenodo-token")] + zenodo_token: Option, + + /// Zenodo host. Uses zenodo.org by default and sandbox.zenodo.org for dev-mode. + #[structopt(long = "zenodo-host")] + zenodo_host: Option, + + /// Community set in Zenodo deposition. + #[structopt(long)] + zenodo_community: Option, + + /// Path to the output file. + #[structopt( + short, + long, + parse(from_os_str), + default_value = "yevis-metadata-uploaded.yml" + )] + output: PathBuf, + + /// Verbose mode. + #[structopt(short, long)] + verbose: bool, + }, } impl Args { @@ -170,6 +211,7 @@ impl Args { Args::Test { verbose, .. } => *verbose, Args::PullRequest { verbose, .. } => *verbose, Args::Publish { verbose, .. } => *verbose, + Args::UploadZenodo { verbose, .. } => *verbose, } } @@ -180,6 +222,7 @@ impl Args { Args::Test { github_token, .. } => github_token.clone(), Args::PullRequest { github_token, .. } => github_token.clone(), Args::Publish { github_token, .. } => github_token.clone(), + Args::UploadZenodo { github_token, .. } => github_token.clone(), } } } diff --git a/src/main.rs b/src/main.rs index 5d21815..20abe8d 100644 --- a/src/main.rs +++ b/src/main.rs @@ -147,6 +147,29 @@ fn main() -> Result<()> { sub_cmd::publish(&meta_vec, &gh_token, &repository, with_test); } + args::Args::UploadZenodo { + metadata_location, + repository, + output, + zenodo_token, + zenodo_host, + zenodo_community, + .. + } => { + let meta_vec = sub_cmd::validate(vec![metadata_location], &gh_token); + let mut meta_loc = meta_vec + .into_iter() + .next() + .expect("Expected at least one metadata"); + sub_cmd::upload_zenodo( + &mut meta_loc, + &output, + &zenodo_token, + &zenodo_host, + &zenodo_community, + &repository, + )?; + } }; Ok(()) } diff --git a/src/sub_cmd.rs b/src/sub_cmd.rs index 2ac0e2e..05112b5 100644 --- a/src/sub_cmd.rs +++ b/src/sub_cmd.rs @@ -4,6 +4,8 @@ pub mod pull_request; pub mod test; pub mod validate; +use crate::zenodo; +use anyhow::bail; use make_template::make_template as make_template_process; use publish::publish as publish_process; use pull_request::pull_request as pull_request_process; @@ -147,3 +149,42 @@ pub fn publish( } }; } + +pub fn upload_zenodo( + meta: &mut metadata::types::Metadata, + output: impl AsRef, + zenodo_token: &Option>, + zenodo_host: &Option>, + zenodo_community: &Option>, + repository: impl AsRef, +) -> Result<(), anyhow::Error> { + info!("{} upload-zenodo", "Running".green()); + let token = match zenodo_token { + Some(zenodo_token) => zenodo_token.as_ref().to_string(), + None => match env::zenodo_token() { + Ok(token) => token, + Err(e) => { + bail!("{} to get Zenodo token with error: {}", "Failed".red(), e); + } + }, + }; + let host = match zenodo_host { + Some(zenodo_host) => zenodo_host.as_ref().to_string(), + None => env::zenodo_host(), + }; + info!( + "Uploading wf_id: {}, version: {} to Zenodo", + meta.id, meta.version + ); + zenodo::upload_zenodo(&host, &token, meta, repository, zenodo_community)?; + info!("Updating workflow metadata to Zenodo URL"); + zenodo::update_metadata(&host, &token, meta)?; + + info!("Writing uploaded metadata to {}", output.as_ref().display()); + let file_ext = metadata::io::parse_file_ext(&output)?; + metadata::io::write_local(meta, &output, &file_ext)?; + + info!("{} upload-zenodo", "Success".green()); + + Ok(()) +} diff --git a/src/sub_cmd/validate.rs b/src/sub_cmd/validate.rs index 0d215b7..fe311ca 100644 --- a/src/sub_cmd/validate.rs +++ b/src/sub_cmd/validate.rs @@ -44,24 +44,24 @@ pub fn validate_version(version: impl AsRef) -> Result<()> { /// Change the license to `spdx_id` /// e.g., `apache-2.0` -> `Apache-2.0` fn validate_license(meta: &mut metadata::types::Metadata, gh_token: impl AsRef) -> Result<()> { - let spdx_id: String = validate_with_github_license_api(gh_token, &meta.license)?; - validate_with_zenodo_license_api(&spdx_id)?; + let (key, spdx_id) = validate_with_github_license_api(gh_token, &meta.license)?; + validate_with_zenodo_license_api(&key)?; meta.license = spdx_id; Ok(()) } #[derive(Debug, Deserialize)] struct LicenseResponse { - permissions: Vec, + key: String, spdx_id: String, } /// https://docs.github.com/ja/rest/reference/licenses#get-a-license -/// Ensure that `distribution` is included in `permissions` field. +/// res: (key, spdx_id), e.g., ("mit", "MIT") fn validate_with_github_license_api( gh_token: impl AsRef, license: impl AsRef, -) -> Result { +) -> Result<(String, String)> { let url = Url::parse(&format!( "https://api.github.com/licenses/{}", license.as_ref() @@ -69,11 +69,7 @@ fn validate_with_github_license_api( let res = gh::get_request(gh_token, &url, &[])?; let res: LicenseResponse = serde_json::from_value(res).context("Failed to parse GitHub license API response")?; - ensure!( - res.permissions.contains(&String::from("distribution")), - "GitHub license API response does not contain `distribution` in `permissions` field" - ); - Ok(res.spdx_id) + Ok((res.key, res.spdx_id)) } /// https://developers.zenodo.org/?shell#retrieve41 diff --git a/src/zenodo.rs b/src/zenodo.rs index 7d86786..072030c 100644 --- a/src/zenodo.rs +++ b/src/zenodo.rs @@ -52,7 +52,7 @@ pub fn upload_zenodo_and_commit_gh( Ok(()) } -fn upload_zenodo( +pub fn upload_zenodo( host: impl AsRef, token: impl AsRef, meta: &mut metadata::types::Metadata, @@ -222,7 +222,7 @@ fn update_deposition_files( Ok(()) } -fn update_metadata( +pub fn update_metadata( host: impl AsRef, token: impl AsRef, meta: &mut metadata::types::Metadata,