From 3855d24439aa252f8d4384a6ebc5b84728a77491 Mon Sep 17 00:00:00 2001 From: Alex Axthelm Date: Fri, 17 May 2024 14:26:08 +0200 Subject: [PATCH 01/21] Add function to inherit parameters from file --- R/parse_json_config.R | 61 ++++++++++++ tests/testthat/test-inherit_config.R | 144 +++++++++++++++++++++++++++ 2 files changed, 205 insertions(+) create mode 100644 R/parse_json_config.R create mode 100644 tests/testthat/test-inherit_config.R diff --git a/R/parse_json_config.R b/R/parse_json_config.R new file mode 100644 index 00000000..e4cc827b --- /dev/null +++ b/R/parse_json_config.R @@ -0,0 +1,61 @@ +parse_params <- function( + params, + inheritence_search_paths +) { + log_trace("Parsing params.") + if (file.exists(params)) { + log_trace("Reading params from file: {params}.}") + } else { + log_trace("Reading params from string.") + } + raw_params <- jsonlite::fromJSON(params) + full_params <- inherit_params( + raw_params, + inheritence_search_paths + ) + return(full_params) +} + +inherit_params <- function( + params, + inheritence_search_paths +) { + inherit_key <- "inherit" + + while (inherit_key %in% names(params)) { + log_trace( + "Key \"{inherit_key}\" found in parameters. Inheriting parameters." + ) + + to_inherit <- params[[inherit_key]] + params[[inherit_key]] <- NULL # remove inherit key + + possible_paths <- file.path( + inheritence_search_paths, + paste0(to_inherit, ".json") + ) + candidate_file <- possible_paths[file.exists(possible_paths)] + if (length(candidate_file) == 0L) { + log_error("Inheritence file not found: {possible_paths}.") + stop("Inheritence file not found.") + } else { + if (length(candidate_file) > 1L) { + log_warn("Multiple files matching inheritence pattern found:") + log_warn("{candidate_file}.") + warning("Multiple inheritence files found.") + browser() + candidate_file <- candidate_file[[1L]] + log_warn("Using first file: {candidate_file}.") + } + } + log_trace("Inheriting parameters from file: {candidate_file}.") + inherit_params <- jsonlite::fromJSON(candidate_file) + params <- merge_lists( + base_list = inherit_params, + overlay_list = params + ) + } + + log_trace("No inheritence key (\"{inherit_key}\") found.") + return(params) +} diff --git a/tests/testthat/test-inherit_config.R b/tests/testthat/test-inherit_config.R new file mode 100644 index 00000000..6fcdc1a5 --- /dev/null +++ b/tests/testthat/test-inherit_config.R @@ -0,0 +1,144 @@ +## save current settings so that we can reset later +threshold <- logger::log_threshold() +appender <- logger::log_appender() +layout <- logger::log_layout() +on.exit({ + ## reset logger settings + logger::log_threshold(threshold) + logger::log_layout(layout) + logger::log_appender(appender) +}) + +logger::log_appender(logger::appender_stdout) +logger::log_threshold(logger::TRACE) +logger::log_layout(logger::layout_simple) + +test_that("No inheritence", { + params <- list( + foo = 1L, + string = "simple params" + ) + results <- inherit_params(params) + expect_identical(results, params) +}) + +test_that("Simple inheritence works", { + params <- list( + foo = 1L, + string = "simple params", + inherit = "test01" + ) + param_dir <- withr::local_tempdir() + writeLines( + '{ + "inherited_key": 2, + "some_other_key": "test01", + "string": "we should not see this" + }', + file.path(param_dir, "test01.json") + ) + results <- inherit_params( + params = params, + inheritence_search_paths = param_dir + ) + expect_identical( + object = results, + expected = list( + inherited_key = 2L, + some_other_key = "test01", + string = "simple params", + foo = 1L + ) + ) +}) + +test_that("Simple inheritence picks the correct file", { + params <- list( + foo = 1L, + string = "simple params", + inherit = "test02" + ) + param_dir <- withr::local_tempdir() + writeLines( + '{ + "inherited_key": 2, + "some_other_key": "test01", + "string": "we should not see this" + }', + file.path(param_dir, "test01.json") + ) + writeLines( + '{ + "inherited_key": 3, + "some_other_key": "test02", + "string": "we should not see this either" + }', + file.path(param_dir, "test02.json") + ) + # Note that we're inheriting from test02.json + results <- inherit_params( + params = params, + inheritence_search_paths = param_dir + ) + expect_identical( + object = results, + expected = list( + inherited_key = 3L, + some_other_key = "test02", + string = "simple params", + foo = 1L + ) + ) +}) + +test_that("Nested inheritence works", { + params <- list( + foo = 1L, + string = "simple params", + inherit = "test01" + ) + param_dir <- withr::local_tempdir() + writeLines( + '{ + "inherited_key": 2, + "some_other_key": "test01", + "string": "we should not see this", + "test01": true, + "inherit": "test02" + }', + file.path(param_dir, "test01.json") + ) + writeLines( + '{ + "inherited_key": 3, + "some_other_key": "test02", + "string": "we should not see this either", + "test02": true + }', + file.path(param_dir, "test02.json") + ) + results <- inherit_params( + params = params, + inheritence_search_paths = param_dir + ) + expect_identical( + object = results, + expected = list( + inherited_key = 2L, + some_other_key = "test01", + string = "simple params", + test02 = TRUE, + test01 = TRUE, + foo = 1L + ) + ) +}) + +# TODO: test not inheriting +# TODO: test not inheriting (through keyname) +# TODO: test inheriting with multiple levels +# TODO: circular inheritance (do not allow!) +# TODO: multiple directories +# TODO: Missing inheritence file +# TODO: multiple named inheritence file +# TODO: multiple "inherit" keys should fail From faa5626a0d895d2627567474da84bcd09c2bc410 Mon Sep 17 00:00:00 2001 From: Alex Axthelm Date: Fri, 17 May 2024 14:55:16 +0200 Subject: [PATCH 02/21] Add handling for multiple inherit keys --- R/parse_json_config.R | 7 ++++ tests/testthat/test-inherit_config.R | 53 +++++++++++++++++++++++++--- 2 files changed, 56 insertions(+), 4 deletions(-) diff --git a/R/parse_json_config.R b/R/parse_json_config.R index e4cc827b..44486714 100644 --- a/R/parse_json_config.R +++ b/R/parse_json_config.R @@ -23,6 +23,13 @@ inherit_params <- function( inherit_key <- "inherit" while (inherit_key %in% names(params)) { + + # check for multiple inheritence keys + if (sum(names(params) == inherit_key) > 1L) { + log_error("Multiple inheritence keys found.") + stop("Multiple inheritence keys found.") + } + log_trace( "Key \"{inherit_key}\" found in parameters. Inheriting parameters." ) diff --git a/tests/testthat/test-inherit_config.R b/tests/testthat/test-inherit_config.R index 6fcdc1a5..0bb36690 100644 --- a/tests/testthat/test-inherit_config.R +++ b/tests/testthat/test-inherit_config.R @@ -134,11 +134,56 @@ test_that("Nested inheritence works", { ) }) -# TODO: test not inheriting -# TODO: test not inheriting (through keyname) -# TODO: test inheriting with multiple levels +test_that("Missing inheritence file throws error", { + params <- list( + foo = 1L, + string = "simple params", + inherit = "test01" + ) + param_dir <- withr::local_tempdir() + testthat::expect_error( + inherit_params( + params = params, + inheritence_search_paths = param_dir + ), + regexp = "^Inheritence file not found.$" + ) +}) + +test_that("Multiple inherit keys in params throws error", { + params <- list( + foo = 1L, + string = "simple params", + inherit = "test01", + inherit = "test02" + ) + param_dir <- withr::local_tempdir() + writeLines( + '{ + "inherited_key": 2, + "some_other_key": "test01", + "string": "we should not see this" + }', + file.path(param_dir, "test01.json") + ) + writeLines( + '{ + "inherited_key": 3, + "some_other_key": "test02", + "string": "we should not see this either" + }', + file.path(param_dir, "test02.json") + ) + testthat::expect_error( + inherit_params( + params = params, + inheritence_search_paths = param_dir + ), + regexp = "^Multiple inheritence keys found.$" + ) +}) + # TODO: circular inheritance (do not allow!) # TODO: multiple directories -# TODO: Missing inheritence file # TODO: multiple named inheritence file # TODO: multiple "inherit" keys should fail From 19eafcc175345896bb25a937d4a4619145bef522 Mon Sep 17 00:00:00 2001 From: Alex Axthelm Date: Fri, 17 May 2024 15:12:43 +0200 Subject: [PATCH 03/21] Handle multiple values in inherit key --- R/parse_json_config.R | 4 ++++ tests/testthat/test-inherit_config.R | 33 +++++++++++++++++++++++++++- 2 files changed, 36 insertions(+), 1 deletion(-) diff --git a/R/parse_json_config.R b/R/parse_json_config.R index 44486714..b903cc70 100644 --- a/R/parse_json_config.R +++ b/R/parse_json_config.R @@ -35,6 +35,10 @@ inherit_params <- function( ) to_inherit <- params[[inherit_key]] + if (length(to_inherit) > 1L) { + log_error("Multiple values in inherit key.") + stop("Multiple values in inherit key.") + } params[[inherit_key]] <- NULL # remove inherit key possible_paths <- file.path( diff --git a/tests/testthat/test-inherit_config.R b/tests/testthat/test-inherit_config.R index 0bb36690..a3186a64 100644 --- a/tests/testthat/test-inherit_config.R +++ b/tests/testthat/test-inherit_config.R @@ -183,7 +183,38 @@ test_that("Multiple inherit keys in params throws error", { ) }) +test_that("Multiple values in inherit key throws error", { + params <- list( + foo = 1L, + string = "simple params", + inherit = c("test01", "test02") + ) + param_dir <- withr::local_tempdir() + writeLines( + '{ + "inherited_key": 2, + "some_other_key": "test01", + "string": "we should not see this" + }', + file.path(param_dir, "test01.json") + ) + writeLines( + '{ + "inherited_key": 3, + "some_other_key": "test02", + "string": "we should not see this either" + }', + file.path(param_dir, "test02.json") + ) + testthat::expect_error( + inherit_params( + params = params, + inheritence_search_paths = param_dir + ), + regexp = "^Multiple values in inherit key.$" + ) +}) + # TODO: circular inheritance (do not allow!) # TODO: multiple directories # TODO: multiple named inheritence file -# TODO: multiple "inherit" keys should fail From a3eb04755e897c2d9f3b8394f75b6481695c5951 Mon Sep 17 00:00:00 2001 From: Alex Axthelm Date: Fri, 17 May 2024 15:20:07 +0200 Subject: [PATCH 04/21] Handle ciruclar inheritence --- R/parse_json_config.R | 9 ++++++++ tests/testthat/test-inherit_config.R | 34 ++++++++++++++++++++++++++++ 2 files changed, 43 insertions(+) diff --git a/R/parse_json_config.R b/R/parse_json_config.R index b903cc70..435a5482 100644 --- a/R/parse_json_config.R +++ b/R/parse_json_config.R @@ -22,6 +22,7 @@ inherit_params <- function( ) { inherit_key <- "inherit" + inherited_files <- NULL while (inherit_key %in% names(params)) { # check for multiple inheritence keys @@ -59,6 +60,14 @@ inherit_params <- function( log_warn("Using first file: {candidate_file}.") } } + if (candidate_file %in% inherited_files) { + log_error( + "Inheritence loop detected while inheriting from {candidate_file}." + ) + log_error("Inherited file: {inherited_files}.") + stop("Inheritence loop detected.") + } + inherited_files <- c(inherited_files, candidate_file) log_trace("Inheriting parameters from file: {candidate_file}.") inherit_params <- jsonlite::fromJSON(candidate_file) params <- merge_lists( diff --git a/tests/testthat/test-inherit_config.R b/tests/testthat/test-inherit_config.R index a3186a64..942963a8 100644 --- a/tests/testthat/test-inherit_config.R +++ b/tests/testthat/test-inherit_config.R @@ -215,6 +215,40 @@ test_that("Multiple values in inherit key throws error", { ) }) +test_that("Circular inheritence throws error", { + params <- list( + foo = 1L, + string = "simple params", + inherit = "test01" + ) + param_dir <- withr::local_tempdir() + writeLines( + '{ + "inherited_key": 2, + "some_other_key": "test01", + "string": "we should not see this", + "inherit": "test02" + }', + file.path(param_dir, "test01.json") + ) + writeLines( + '{ + "inherited_key": 3, + "some_other_key": "test02", + "string": "we should not see this either", + "inherit": "test01" + }', + file.path(param_dir, "test02.json") + ) + testthat::expect_error( + inherit_params( + params = params, + inheritence_search_paths = param_dir + ), + regexp = "^Inheritence loop detected.$" + ) +}) + # TODO: circular inheritance (do not allow!) # TODO: multiple directories # TODO: multiple named inheritence file From 5adb6848532d8d532e5d6f4054f61f4c9d404831 Mon Sep 17 00:00:00 2001 From: Alex Axthelm Date: Fri, 17 May 2024 15:40:31 +0200 Subject: [PATCH 05/21] test for inheritence across search path --- R/parse_json_config.R | 1 - tests/testthat/test-inherit_config.R | 58 +++++++++++++++++++++++++--- 2 files changed, 52 insertions(+), 7 deletions(-) diff --git a/R/parse_json_config.R b/R/parse_json_config.R index 435a5482..ca58ac69 100644 --- a/R/parse_json_config.R +++ b/R/parse_json_config.R @@ -55,7 +55,6 @@ inherit_params <- function( log_warn("Multiple files matching inheritence pattern found:") log_warn("{candidate_file}.") warning("Multiple inheritence files found.") - browser() candidate_file <- candidate_file[[1L]] log_warn("Using first file: {candidate_file}.") } diff --git a/tests/testthat/test-inherit_config.R b/tests/testthat/test-inherit_config.R index 942963a8..9b7029a6 100644 --- a/tests/testthat/test-inherit_config.R +++ b/tests/testthat/test-inherit_config.R @@ -49,7 +49,7 @@ test_that("Simple inheritence works", { string = "simple params", foo = 1L ) - ) + ) }) test_that("Simple inheritence picks the correct file", { @@ -88,7 +88,7 @@ test_that("Simple inheritence picks the correct file", { string = "simple params", foo = 1L ) - ) + ) }) test_that("Nested inheritence works", { @@ -131,7 +131,7 @@ test_that("Nested inheritence works", { test01 = TRUE, foo = 1L ) - ) + ) }) test_that("Missing inheritence file throws error", { @@ -155,7 +155,7 @@ test_that("Multiple inherit keys in params throws error", { foo = 1L, string = "simple params", inherit = "test01", - inherit = "test02" + inherit = "test02" # nolint: duplicate_argument_linter ) param_dir <- withr::local_tempdir() writeLines( @@ -249,6 +249,52 @@ test_that("Circular inheritence throws error", { ) }) -# TODO: circular inheritance (do not allow!) -# TODO: multiple directories +test_that("Searching across multiple directories works", { + params <- list( + foo = 1L, + string = "simple params", + inherit = "test01" + ) + first_dir <- withr::local_tempdir() + writeLines( + '{ + "inherited_key": 2, + "dir": "first", + "some_other_key": "test01", + "string": "we should not see this", + "test01": true, + "inherit": "test02" + }', + file.path(first_dir, "test01.json") + ) + second_dir <- withr::local_tempdir() + writeLines( + '{ + "inherited_key": 3, + "dir": "second", + "some_other_key": "test02", + "string": "we should not see this either", + "test02": true + }', + file.path(second_dir, "test02.json") + ) + results <- inherit_params( + params = params, + inheritence_search_paths = c(first_dir, second_dir) + ) + expect_identical( + object = results, + expected = list( + inherited_key = 2L, + dir = "first", # inheriting from first_dir/test01.json + some_other_key = "test01", + string = "simple params", + test02 = TRUE, + test01 = TRUE, + foo = 1L + ) + ) +}) + + # TODO: multiple named inheritence file From 59bca21cd3d4489d0b52e52951877f75a471b596 Mon Sep 17 00:00:00 2001 From: Alex Axthelm Date: Fri, 17 May 2024 15:47:03 +0200 Subject: [PATCH 06/21] Add test for warning when multiple candidate files --- tests/testthat/test-inherit_config.R | 50 +++++++++++++++++++++++++++- 1 file changed, 49 insertions(+), 1 deletion(-) diff --git a/tests/testthat/test-inherit_config.R b/tests/testthat/test-inherit_config.R index 9b7029a6..5f79bd37 100644 --- a/tests/testthat/test-inherit_config.R +++ b/tests/testthat/test-inherit_config.R @@ -297,4 +297,52 @@ test_that("Searching across multiple directories works", { }) -# TODO: multiple named inheritence file +test_that("Searching across multiple directories works", { + params <- list( + foo = 1L, + string = "simple params", + inherit = "test01" + ) + first_dir <- withr::local_tempdir() + writeLines( + '{ + "inherited_key": 2, + "dir": "first", + "some_other_key": "test01", + "string": "we should not see this", + "test01": true + }', + file.path(first_dir, "test01.json") + ) + second_dir <- withr::local_tempdir() + writeLines( + '{ + "inherited_key": 3, + "dir": "second", + "some_other_key": "secret bonus file", + "string": "we should not see this either", + "test02": true + }', + file.path(second_dir, "test01.json") + ) + testthat::expect_warning( + { + results <- inherit_params( + params = params, + inheritence_search_paths = c(first_dir, second_dir) + ) + }, + regexp = "^Multiple inheritence files found.$" + ) + expect_identical( + object = results, + expected = list( + inherited_key = 2L, + dir = "first", # inheriting from first_dir/test01.json + some_other_key = "test01", + string = "simple params", + test01 = TRUE, + foo = 1L + ) + ) +}) From 6eb5d1ad25a2f7d9c49c5c4255c35dc8c84f9248 Mon Sep 17 00:00:00 2001 From: Alex Axthelm Date: Fri, 17 May 2024 16:30:23 +0200 Subject: [PATCH 07/21] Rename file --- R/{parse_json_config.R => parse_json_params.R} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename R/{parse_json_config.R => parse_json_params.R} (100%) diff --git a/R/parse_json_config.R b/R/parse_json_params.R similarity index 100% rename from R/parse_json_config.R rename to R/parse_json_params.R From eff0680c1d78a8de56753bd92d7b523eaa36ba7a Mon Sep 17 00:00:00 2001 From: Alex Axthelm Date: Fri, 17 May 2024 16:32:42 +0200 Subject: [PATCH 08/21] change log level in testing file --- tests/testthat/test-inherit_config.R | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/testthat/test-inherit_config.R b/tests/testthat/test-inherit_config.R index 5f79bd37..98657062 100644 --- a/tests/testthat/test-inherit_config.R +++ b/tests/testthat/test-inherit_config.R @@ -10,7 +10,7 @@ on.exit({ }) logger::log_appender(logger::appender_stdout) -logger::log_threshold(logger::TRACE) +logger::log_threshold(logger::FATAL) logger::log_layout(logger::layout_simple) test_that("No inheritence", { From 6e969ff893cde5af602ddb57ef8e755ce144cf6a Mon Sep 17 00:00:00 2001 From: Alex Axthelm Date: Fri, 17 May 2024 16:39:12 +0200 Subject: [PATCH 09/21] Add test for only inheriting --- tests/testthat/test-inherit_config.R | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/tests/testthat/test-inherit_config.R b/tests/testthat/test-inherit_config.R index 98657062..ffa59fb8 100644 --- a/tests/testthat/test-inherit_config.R +++ b/tests/testthat/test-inherit_config.R @@ -52,6 +52,33 @@ test_that("Simple inheritence works", { ) }) +test_that("Only inheritence works", { + params <- list( + inherit = "test01" + ) + param_dir <- withr::local_tempdir() + writeLines( + '{ + "inherited_key": 2, + "some_other_key": "test01", + "string": "we should not see this" + }', + file.path(param_dir, "test01.json") + ) + results <- inherit_params( + params = params, + inheritence_search_paths = param_dir + ) + expect_identical( + object = results, + expected = list( + inherited_key = 2L, + some_other_key = "test01", + string = "we should not see this" + ) + ) +}) + test_that("Simple inheritence picks the correct file", { params <- list( foo = 1L, From 4dc4d4a05ae305897beaf2bf6266f3ada72745a3 Mon Sep 17 00:00:00 2001 From: Alex Axthelm Date: Mon, 20 May 2024 12:51:23 +0200 Subject: [PATCH 10/21] Version bump --- DESCRIPTION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/DESCRIPTION b/DESCRIPTION index 6c74f065..9933a386 100644 --- a/DESCRIPTION +++ b/DESCRIPTION @@ -1,6 +1,6 @@ Package: pacta.workflow.utils Title: Utility functions for PACTA workflows -Version: 0.0.0.9004 +Version: 0.0.0.9005 Authors@R: c(person(given = "Alex", family = "Axthelm", From 0a78ce31dd5652ce7ed8d39e8e3fc0397d3e5891 Mon Sep 17 00:00:00 2001 From: Alex Axthelm Date: Mon, 20 May 2024 12:52:03 +0200 Subject: [PATCH 11/21] Update variable name --- R/parse_json_params.R | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/R/parse_json_params.R b/R/parse_json_params.R index ca58ac69..e964e9a9 100644 --- a/R/parse_json_params.R +++ b/R/parse_json_params.R @@ -1,14 +1,14 @@ parse_params <- function( - params, + JSON, inheritence_search_paths ) { log_trace("Parsing params.") - if (file.exists(params)) { - log_trace("Reading params from file: {params}.}") + if (file.exists(JSON)) { + log_trace("Reading params from file: {JSON}.}") } else { log_trace("Reading params from string.") } - raw_params <- jsonlite::fromJSON(params) + raw_params <- jsonlite::fromJSON(JSON) full_params <- inherit_params( raw_params, inheritence_search_paths From 934ab26738e52f6ea0d1d47991626bc021342791 Mon Sep 17 00:00:00 2001 From: Alex Axthelm Date: Mon, 20 May 2024 13:02:08 +0200 Subject: [PATCH 12/21] Add helper for test to test commandArgs --- tests/testthat/helper-run_with_cmd_args.R | 22 ++++ tests/testthat/test-parse_params.R | 138 ++++++++++++++++++++++ 2 files changed, 160 insertions(+) create mode 100644 tests/testthat/helper-run_with_cmd_args.R create mode 100644 tests/testthat/test-parse_params.R diff --git a/tests/testthat/helper-run_with_cmd_args.R b/tests/testthat/helper-run_with_cmd_args.R new file mode 100644 index 00000000..b56429be --- /dev/null +++ b/tests/testthat/helper-run_with_cmd_args.R @@ -0,0 +1,22 @@ +run_with_cmd_args <- function( + code, + cmdargs = character() +) { + script <- withr::local_tempfile(fileext = ".R") + results_file <- withr::local_tempfile(fileext = ".RDS") + writeLines( + text = c( + "args <- commandArgs(trailingOnly = TRUE)", + "output <- pacta.workflow.utils:::parse_params(args[1])", + paste0("saveRDS(output, '", results_file, "')") + ), + con = script + ) + readLines(script) + callr::rscript( + script = script, + cmdargs = cmdargs + ) + output <- readRDS(results_file) + return(output) +} diff --git a/tests/testthat/test-parse_params.R b/tests/testthat/test-parse_params.R new file mode 100644 index 00000000..f6dd5144 --- /dev/null +++ b/tests/testthat/test-parse_params.R @@ -0,0 +1,138 @@ +## save current settings so that we can reset later +threshold <- logger::log_threshold() +appender <- logger::log_appender() +layout <- logger::log_layout() +on.exit({ + ## reset logger settings + logger::log_threshold(threshold) + logger::log_layout(layout) + logger::log_appender(appender) +}) + +logger::log_appender(logger::appender_stdout) +logger::log_threshold(logger::FATAL) +logger::log_layout(logger::layout_simple) + + +test_that("No inheritence, pass as string", { + json_string <- '{ + "id": 1, + "name": "A green door", + "price": 12.50, + "tags": ["home", "green"] + }' + results <- parse_params(json_string) + expect_identical( + object = results, + expected = list( + id = 1L, + name = "A green door", + price = 12.5, + tags = c("home", "green") + ) + ) +}) + +test_that("No inheritence, pass as file", { + json_string <- '{ + "id": 1, + "name": "A green door", + "price": 12.50, + "tags": ["home", "green"] + }' + json_file <- withr::local_tempfile(fileext = ".json") + writeLines(json_string, json_file) + results <- parse_params(json_file) + expect_identical( + object = results, + expected = list( + id = 1L, + name = "A green door", + price = 12.5, + tags = c("home", "green") + ) + ) +}) + +test_that("No inheritence, pass string to command args", { + json_string <- '{ + "id": 1, + "name": "A green door", + "price": 12.50, + "tags": ["home", "green"] + }' |> + gsub(pattern = "\\s{2,}", replacement = "", x = _) # Note _ instead of . for base pipe + + results <- run_with_cmd_args( + code = "pacta.workflow.utils:::parse_params(commandArgs())", + cmdargs = json_string + ) + expect_identical( + object = results, + expected = list( + id = 1L, + name = "A green door", + price = 12.5, + tags = c("home", "green") + ) + ) +}) + + + +base_params_dir <- withr::local_tempdir() +base_01_string <- '{ + "name": "A green door", + "tags": ["home", "green"], + "supplier": "ACME Doors" + }' +writeLines( + base_01_string, + file.path(base_params_dir, "base01.json") +) + +test_that("Simple inheritence, pass as string", { + json_string <- '{ + "id": 1, + "price": 12.50, + "inherit": "base01" + }' + results <- parse_params( + JSON = json_string, + inheritence_search_paths = base_params_dir + ) + expect_identical( + object = results, + expected = list( + name = "A green door", + tags = c("home", "green"), + supplier = "ACME Doors", + id = 1L, + price = 12.5 + ) + ) +}) + +test_that("Simple inheritence, pass as file", { + json_string <- '{ + "id": 1, + "price": 12.50, + "inherit": "base01" + }' + json_file <- withr::local_tempfile(fileext = ".json") + writeLines(json_string, json_file) + results <- parse_params( + JSON = json_file, + inheritence_search_paths = base_params_dir + ) + expect_identical( + object = results, + expected = list( + name = "A green door", + tags = c("home", "green"), + supplier = "ACME Doors", + id = 1L, + price = 12.5 + ) + ) +}) From 5f84cd5a0d29e73ca5d3c406f2a8efd7d62cd122 Mon Sep 17 00:00:00 2001 From: Alex Axthelm Date: Mon, 20 May 2024 13:04:57 +0200 Subject: [PATCH 13/21] More test with cmdargs --- tests/testthat/test-parse_params.R | 47 ++++++++++++++++++++++++++++-- 1 file changed, 45 insertions(+), 2 deletions(-) diff --git a/tests/testthat/test-parse_params.R b/tests/testthat/test-parse_params.R index f6dd5144..fdee12bb 100644 --- a/tests/testthat/test-parse_params.R +++ b/tests/testthat/test-parse_params.R @@ -54,7 +54,7 @@ test_that("No inheritence, pass as file", { ) }) -test_that("No inheritence, pass string to command args", { +test_that("No inheritence, pass oneline string to command args", { json_string <- '{ "id": 1, "name": "A green door", @@ -62,7 +62,6 @@ test_that("No inheritence, pass string to command args", { "tags": ["home", "green"] }' |> gsub(pattern = "\\s{2,}", replacement = "", x = _) # Note _ instead of . for base pipe - results <- run_with_cmd_args( code = "pacta.workflow.utils:::parse_params(commandArgs())", cmdargs = json_string @@ -78,7 +77,51 @@ test_that("No inheritence, pass string to command args", { ) }) +test_that("No inheritence, pass filepath to command args", { + json_string <- '{ + "id": 1, + "name": "A green door", + "price": 12.50, + "tags": ["home", "green"] + }' + json_file <- withr::local_tempfile(fileext = ".json") + writeLines(json_string, json_file) + results <- run_with_cmd_args( + code = "pacta.workflow.utils:::parse_params(commandArgs())", + cmdargs = json_file + ) + expect_identical( + object = results, + expected = list( + id = 1L, + name = "A green door", + price = 12.5, + tags = c("home", "green") + ) + ) +}) +test_that("No inheritence, pass multiline string to command args", { + json_string <- '{ + "id": 1, + "name": "A green door", + "price": 12.50, + "tags": ["home", "green"] + }' + results <- run_with_cmd_args( + code = "pacta.workflow.utils:::parse_params(commandArgs())", + cmdargs = json_string + ) + expect_identical( + object = results, + expected = list( + id = 1L, + name = "A green door", + price = 12.5, + tags = c("home", "green") + ) + ) +}) base_params_dir <- withr::local_tempdir() base_01_string <- '{ From 9246e7c9d7294657cd4701ea779de19d12a6ab54 Mon Sep 17 00:00:00 2001 From: Alex Axthelm Date: Mon, 20 May 2024 13:42:06 +0200 Subject: [PATCH 14/21] Abandon testing commandArgs --- R/parse_json_params.R | 2 +- tests/testthat/helper-run_with_cmd_args.R | 22 -------- tests/testthat/test-parse_params.R | 69 ----------------------- 3 files changed, 1 insertion(+), 92 deletions(-) delete mode 100644 tests/testthat/helper-run_with_cmd_args.R diff --git a/R/parse_json_params.R b/R/parse_json_params.R index e964e9a9..3d7ca3a7 100644 --- a/R/parse_json_params.R +++ b/R/parse_json_params.R @@ -1,6 +1,6 @@ parse_params <- function( JSON, - inheritence_search_paths + inheritence_search_paths = NULL ) { log_trace("Parsing params.") if (file.exists(JSON)) { diff --git a/tests/testthat/helper-run_with_cmd_args.R b/tests/testthat/helper-run_with_cmd_args.R deleted file mode 100644 index b56429be..00000000 --- a/tests/testthat/helper-run_with_cmd_args.R +++ /dev/null @@ -1,22 +0,0 @@ -run_with_cmd_args <- function( - code, - cmdargs = character() -) { - script <- withr::local_tempfile(fileext = ".R") - results_file <- withr::local_tempfile(fileext = ".RDS") - writeLines( - text = c( - "args <- commandArgs(trailingOnly = TRUE)", - "output <- pacta.workflow.utils:::parse_params(args[1])", - paste0("saveRDS(output, '", results_file, "')") - ), - con = script - ) - readLines(script) - callr::rscript( - script = script, - cmdargs = cmdargs - ) - output <- readRDS(results_file) - return(output) -} diff --git a/tests/testthat/test-parse_params.R b/tests/testthat/test-parse_params.R index fdee12bb..e635113e 100644 --- a/tests/testthat/test-parse_params.R +++ b/tests/testthat/test-parse_params.R @@ -54,75 +54,6 @@ test_that("No inheritence, pass as file", { ) }) -test_that("No inheritence, pass oneline string to command args", { - json_string <- '{ - "id": 1, - "name": "A green door", - "price": 12.50, - "tags": ["home", "green"] - }' |> - gsub(pattern = "\\s{2,}", replacement = "", x = _) # Note _ instead of . for base pipe - results <- run_with_cmd_args( - code = "pacta.workflow.utils:::parse_params(commandArgs())", - cmdargs = json_string - ) - expect_identical( - object = results, - expected = list( - id = 1L, - name = "A green door", - price = 12.5, - tags = c("home", "green") - ) - ) -}) - -test_that("No inheritence, pass filepath to command args", { - json_string <- '{ - "id": 1, - "name": "A green door", - "price": 12.50, - "tags": ["home", "green"] - }' - json_file <- withr::local_tempfile(fileext = ".json") - writeLines(json_string, json_file) - results <- run_with_cmd_args( - code = "pacta.workflow.utils:::parse_params(commandArgs())", - cmdargs = json_file - ) - expect_identical( - object = results, - expected = list( - id = 1L, - name = "A green door", - price = 12.5, - tags = c("home", "green") - ) - ) -}) - -test_that("No inheritence, pass multiline string to command args", { - json_string <- '{ - "id": 1, - "name": "A green door", - "price": 12.50, - "tags": ["home", "green"] - }' - results <- run_with_cmd_args( - code = "pacta.workflow.utils:::parse_params(commandArgs())", - cmdargs = json_string - ) - expect_identical( - object = results, - expected = list( - id = 1L, - name = "A green door", - price = 12.5, - tags = c("home", "green") - ) - ) -}) - base_params_dir <- withr::local_tempdir() base_01_string <- '{ "name": "A green door", From b84ed4e1e04d6a677733422fd0d36a3c109e35f4 Mon Sep 17 00:00:00 2001 From: Alex Axthelm Date: Mon, 20 May 2024 13:43:00 +0200 Subject: [PATCH 15/21] Match case for variable name --- R/parse_json_params.R | 8 ++++---- tests/testthat/test-parse_params.R | 4 ++-- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/R/parse_json_params.R b/R/parse_json_params.R index 3d7ca3a7..b1d3de6e 100644 --- a/R/parse_json_params.R +++ b/R/parse_json_params.R @@ -1,14 +1,14 @@ parse_params <- function( - JSON, + json, inheritence_search_paths = NULL ) { log_trace("Parsing params.") - if (file.exists(JSON)) { - log_trace("Reading params from file: {JSON}.}") + if (file.exists(json)) { + log_trace("Reading params from file: {json}.}") } else { log_trace("Reading params from string.") } - raw_params <- jsonlite::fromJSON(JSON) + raw_params <- jsonlite::fromJSON(json) full_params <- inherit_params( raw_params, inheritence_search_paths diff --git a/tests/testthat/test-parse_params.R b/tests/testthat/test-parse_params.R index e635113e..0fedd3c4 100644 --- a/tests/testthat/test-parse_params.R +++ b/tests/testthat/test-parse_params.R @@ -72,7 +72,7 @@ test_that("Simple inheritence, pass as string", { "inherit": "base01" }' results <- parse_params( - JSON = json_string, + json = json_string, inheritence_search_paths = base_params_dir ) expect_identical( @@ -96,7 +96,7 @@ test_that("Simple inheritence, pass as file", { json_file <- withr::local_tempfile(fileext = ".json") writeLines(json_string, json_file) results <- parse_params( - JSON = json_file, + json = json_file, inheritence_search_paths = base_params_dir ) expect_identical( From c7a738eb181a9c187752d4f1817981ef55fe25f7 Mon Sep 17 00:00:00 2001 From: Alex Axthelm Date: Mon, 20 May 2024 14:59:11 +0200 Subject: [PATCH 16/21] Add JSON validation --- R/parse_json_params.R | 51 ++++++++++++++++++++- tests/testthat/test-parse_params.R | 73 ++++++++++++++++++++++++++++++ 2 files changed, 123 insertions(+), 1 deletion(-) diff --git a/R/parse_json_params.R b/R/parse_json_params.R index b1d3de6e..7739fb38 100644 --- a/R/parse_json_params.R +++ b/R/parse_json_params.R @@ -1,6 +1,7 @@ parse_params <- function( json, - inheritence_search_paths = NULL + inheritence_search_paths = NULL, + schema_file = NULL ) { log_trace("Parsing params.") if (file.exists(json)) { @@ -13,6 +14,31 @@ parse_params <- function( raw_params, inheritence_search_paths ) + + if (!is.null(schema_file)) { + if (requireNamespace("jsonvalidate", quietly = TRUE)) { + log_trace("Validating parameters.") + validator <- jsonvalidate::json_schema[["new"]](schema = schema_file) + validation_results <- validator[["validate"]]( + json = jsonlite::toJSON(full_params, auto_unbox = TRUE), + verbose = TRUE + ) + if (validation_results) { + log_trace("Validation successful.") + } else { + log_error("Validation against JSON Schema failed.") + log_error("Schema file: {schema_file}") + pretty_log_jsonvalidate_errors(validation_results) + stop("JSON Validation failed.") + } + } else { + log_error("jsonvalidate package not found.") + stop("jsonvalidate package not found.") + } + } else { + log_trace("No JSON Schema provided. Skipping validation.") + } + return(full_params) } @@ -78,3 +104,26 @@ inherit_params <- function( log_trace("No inheritence key (\"{inherit_key}\") found.") return(params) } + +pretty_log_jsonvalidate_errors <- function( + validation_object, + logging_function = log_error + ) { + errors <- attr(validation_object, "errors") + if (length(errors) == 0L) { + return(NULL) + } + for (row in seq(1, nrow(errors))) { + error <- errors[row, ] + error_message <- error[["message"]] + instance_path <- error[["instancePath"]] + keyword <- error[["keyword"]] + schema_path <- error[["schemaPath"]] + logging_function("JSON Validation Error ({row} / {nrow(errors)}):") + logging_function(" Keyword: {keyword}") + logging_function(" instancePath: {instance_path}") + logging_function(" schemaPath: {schema_path}") + logging_function(" Message: {error_message}") + } + return(errors) +} diff --git a/tests/testthat/test-parse_params.R b/tests/testthat/test-parse_params.R index 0fedd3c4..eb221e1c 100644 --- a/tests/testthat/test-parse_params.R +++ b/tests/testthat/test-parse_params.R @@ -110,3 +110,76 @@ test_that("Simple inheritence, pass as file", { ) ) }) + +product_schema <- '{ + "$schema": "http://json-schema.org/draft-04/schema#", + "title": "Product", + "description": "A product from Acme\'s catalog", + "type": "object", + "properties": { + "id": { + "description": "The unique identifier for a product", + "type": "integer" + }, + "name": { + "description": "Name of the product", + "type": "string" + }, + "price": { + "type": "number", + "minimum": 0, + "exclusiveMinimum": true + }, + "tags": { + "type": "array", + "items": { + "type": "string" + }, + "minItems": 1, + "uniqueItems": true + } + }, + "required": ["id", "name", "price"] +}' +schema_dir <- withr::local_tempdir() +schema_file <- file.path(schema_dir, "product.json") +writeLines(product_schema, schema_file) + +test_that("No inheritence, pass as string, validation works", { + json_string <- '{ + "id": 1, + "name": "A green door", + "price": 12.50, + "tags": ["home", "green"] + }' + results <- parse_params( + json = json_string, + schema_file = schema_file + ) + expect_identical( + object = results, + expected = list( + id = 1L, + name = "A green door", + price = 12.5, + tags = c("home", "green") + ) + ) +}) + +test_that("No inheritence, pass as string, failing validation works", { + json_string <- '{ + "id": 1.5, + "price": 12.50, + "tags": ["home", "green"] + }' + testthat::expect_error( + object = { + parse_params( + json = json_string, + schema_file = schema_file + ) + }, + regexp = "^JSON Validation failed.$" + ) +}) From 75332b1041b46304e95bbdc78887edfc71f6fd8f Mon Sep 17 00:00:00 2001 From: Alex Axthelm Date: Mon, 20 May 2024 15:09:29 +0200 Subject: [PATCH 17/21] simplify json error logging --- R/parse_json_params.R | 17 ++++++----------- 1 file changed, 6 insertions(+), 11 deletions(-) diff --git a/R/parse_json_params.R b/R/parse_json_params.R index 7739fb38..761c7f59 100644 --- a/R/parse_json_params.R +++ b/R/parse_json_params.R @@ -108,22 +108,17 @@ inherit_params <- function( pretty_log_jsonvalidate_errors <- function( validation_object, logging_function = log_error - ) { +) { errors <- attr(validation_object, "errors") if (length(errors) == 0L) { return(NULL) } - for (row in seq(1, nrow(errors))) { - error <- errors[row, ] - error_message <- error[["message"]] - instance_path <- error[["instancePath"]] - keyword <- error[["keyword"]] - schema_path <- error[["schemaPath"]] + for (row in seq(1L, nrow(errors))) { logging_function("JSON Validation Error ({row} / {nrow(errors)}):") - logging_function(" Keyword: {keyword}") - logging_function(" instancePath: {instance_path}") - logging_function(" schemaPath: {schema_path}") - logging_function(" Message: {error_message}") + logging_function(" Keyword: {errors[[row, 'keyword']]}") + logging_function(" instancePath: {errors[[row, 'instancePath']]}") + logging_function(" schemaPath: {errors[[row, 'schemaPath']]}") + logging_function(" Message: {errors[[row, 'message']]}") } return(errors) } From 9072af246272489c777112e52523189ee183d7b9 Mon Sep 17 00:00:00 2001 From: Alex Axthelm Date: Mon, 20 May 2024 15:10:11 +0200 Subject: [PATCH 18/21] linting --- tests/testthat/test-parse_params.R | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/tests/testthat/test-parse_params.R b/tests/testthat/test-parse_params.R index eb221e1c..40bb7d0c 100644 --- a/tests/testthat/test-parse_params.R +++ b/tests/testthat/test-parse_params.R @@ -175,11 +175,11 @@ test_that("No inheritence, pass as string, failing validation works", { }' testthat::expect_error( object = { - parse_params( - json = json_string, - schema_file = schema_file - ) - }, - regexp = "^JSON Validation failed.$" + parse_params( + json = json_string, + schema_file = schema_file + ) + }, + regexp = "^JSON Validation failed.$" ) }) From a789216106e99700328082f4354ec73e73864ec8 Mon Sep 17 00:00:00 2001 From: Alex Axthelm Date: Mon, 20 May 2024 15:12:14 +0200 Subject: [PATCH 19/21] Add dependency to DESCRIPTION --- DESCRIPTION | 1 + 1 file changed, 1 insertion(+) diff --git a/DESCRIPTION b/DESCRIPTION index 9933a386..c57f337f 100644 --- a/DESCRIPTION +++ b/DESCRIPTION @@ -25,6 +25,7 @@ Imports: Suggests: covr, devtools, + jsonvalidate, pak, testthat (>= 3.0.0), withr From 9870c14b1ce6daeb8ceb997ebce8e7669ede6ab4 Mon Sep 17 00:00:00 2001 From: Alex Axthelm Date: Mon, 20 May 2024 15:14:45 +0200 Subject: [PATCH 20/21] Add test for inheritence and schema validation --- tests/testthat/test-parse_params.R | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/tests/testthat/test-parse_params.R b/tests/testthat/test-parse_params.R index 40bb7d0c..93ac9266 100644 --- a/tests/testthat/test-parse_params.R +++ b/tests/testthat/test-parse_params.R @@ -183,3 +183,26 @@ test_that("No inheritence, pass as string, failing validation works", { regexp = "^JSON Validation failed.$" ) }) + +test_that("simple inheritence, pass as string, validation works", { + json_string <- '{ + "id": 1, + "price": 12.50, + "inherit": "base01" + }' + results <- parse_params( + json = json_string, + inheritence_search_paths = base_params_dir, + schema_file = schema_file + ) + expect_identical( + object = results, + expected = list( + name = "A green door", + tags = c("home", "green"), + supplier = "ACME Doors", + id = 1L, + price = 12.5 + ) + ) +}) From 9c3f6aaa631ab980288840f3885620a701c88a11 Mon Sep 17 00:00:00 2001 From: Alex Axthelm Date: Mon, 20 May 2024 15:52:01 +0200 Subject: [PATCH 21/21] Syntax form CRAN Version of `jsonvalidate` --- R/parse_json_params.R | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/R/parse_json_params.R b/R/parse_json_params.R index 761c7f59..466279f8 100644 --- a/R/parse_json_params.R +++ b/R/parse_json_params.R @@ -18,9 +18,9 @@ parse_params <- function( if (!is.null(schema_file)) { if (requireNamespace("jsonvalidate", quietly = TRUE)) { log_trace("Validating parameters.") - validator <- jsonvalidate::json_schema[["new"]](schema = schema_file) - validation_results <- validator[["validate"]]( + validation_results <- jsonvalidate::json_validate( json = jsonlite::toJSON(full_params, auto_unbox = TRUE), + schema = schema_file, verbose = TRUE ) if (validation_results) { @@ -114,7 +114,7 @@ pretty_log_jsonvalidate_errors <- function( return(NULL) } for (row in seq(1L, nrow(errors))) { - logging_function("JSON Validation Error ({row} / {nrow(errors)}):") + logging_function("JSON Validation ({row} / {nrow(errors)}):") logging_function(" Keyword: {errors[[row, 'keyword']]}") logging_function(" instancePath: {errors[[row, 'instancePath']]}") logging_function(" schemaPath: {errors[[row, 'schemaPath']]}")