diff --git a/DESCRIPTION b/DESCRIPTION index 6c74f065..c57f337f 100644 --- a/DESCRIPTION +++ b/DESCRIPTION @@ -1,6 +1,6 @@ Package: pacta.workflow.utils Title: Utility functions for PACTA workflows -Version: 0.0.0.9004 +Version: 0.0.0.9005 Authors@R: c(person(given = "Alex", family = "Axthelm", @@ -25,6 +25,7 @@ Imports: Suggests: covr, devtools, + jsonvalidate, pak, testthat (>= 3.0.0), withr diff --git a/R/parse_json_params.R b/R/parse_json_params.R new file mode 100644 index 00000000..466279f8 --- /dev/null +++ b/R/parse_json_params.R @@ -0,0 +1,124 @@ +parse_params <- function( + json, + inheritence_search_paths = NULL, + schema_file = NULL +) { + log_trace("Parsing params.") + if (file.exists(json)) { + log_trace("Reading params from file: {json}.}") + } else { + log_trace("Reading params from string.") + } + raw_params <- jsonlite::fromJSON(json) + full_params <- inherit_params( + raw_params, + inheritence_search_paths + ) + + if (!is.null(schema_file)) { + if (requireNamespace("jsonvalidate", quietly = TRUE)) { + log_trace("Validating parameters.") + validation_results <- jsonvalidate::json_validate( + json = jsonlite::toJSON(full_params, auto_unbox = TRUE), + schema = schema_file, + verbose = TRUE + ) + if (validation_results) { + log_trace("Validation successful.") + } else { + log_error("Validation against JSON Schema failed.") + log_error("Schema file: {schema_file}") + pretty_log_jsonvalidate_errors(validation_results) + stop("JSON Validation failed.") + } + } else { + log_error("jsonvalidate package not found.") + stop("jsonvalidate package not found.") + } + } else { + log_trace("No JSON Schema provided. Skipping validation.") + } + + return(full_params) +} + +inherit_params <- function( + params, + inheritence_search_paths +) { + inherit_key <- "inherit" + + inherited_files <- NULL + while (inherit_key %in% names(params)) { + + # check for multiple inheritence keys + if (sum(names(params) == inherit_key) > 1L) { + log_error("Multiple inheritence keys found.") + stop("Multiple inheritence keys found.") + } + + log_trace( + "Key \"{inherit_key}\" found in parameters. Inheriting parameters." + ) + + to_inherit <- params[[inherit_key]] + if (length(to_inherit) > 1L) { + log_error("Multiple values in inherit key.") + stop("Multiple values in inherit key.") + } + params[[inherit_key]] <- NULL # remove inherit key + + possible_paths <- file.path( + inheritence_search_paths, + paste0(to_inherit, ".json") + ) + candidate_file <- possible_paths[file.exists(possible_paths)] + if (length(candidate_file) == 0L) { + log_error("Inheritence file not found: {possible_paths}.") + stop("Inheritence file not found.") + } else { + if (length(candidate_file) > 1L) { + log_warn("Multiple files matching inheritence pattern found:") + log_warn("{candidate_file}.") + warning("Multiple inheritence files found.") + candidate_file <- candidate_file[[1L]] + log_warn("Using first file: {candidate_file}.") + } + } + if (candidate_file %in% inherited_files) { + log_error( + "Inheritence loop detected while inheriting from {candidate_file}." + ) + log_error("Inherited file: {inherited_files}.") + stop("Inheritence loop detected.") + } + inherited_files <- c(inherited_files, candidate_file) + log_trace("Inheriting parameters from file: {candidate_file}.") + inherit_params <- jsonlite::fromJSON(candidate_file) + params <- merge_lists( + base_list = inherit_params, + overlay_list = params + ) + } + + log_trace("No inheritence key (\"{inherit_key}\") found.") + return(params) +} + +pretty_log_jsonvalidate_errors <- function( + validation_object, + logging_function = log_error +) { + errors <- attr(validation_object, "errors") + if (length(errors) == 0L) { + return(NULL) + } + for (row in seq(1L, nrow(errors))) { + logging_function("JSON Validation ({row} / {nrow(errors)}):") + logging_function(" Keyword: {errors[[row, 'keyword']]}") + logging_function(" instancePath: {errors[[row, 'instancePath']]}") + logging_function(" schemaPath: {errors[[row, 'schemaPath']]}") + logging_function(" Message: {errors[[row, 'message']]}") + } + return(errors) +} diff --git a/tests/testthat/test-inherit_config.R b/tests/testthat/test-inherit_config.R new file mode 100644 index 00000000..ffa59fb8 --- /dev/null +++ b/tests/testthat/test-inherit_config.R @@ -0,0 +1,375 @@ +## save current settings so that we can reset later +threshold <- logger::log_threshold() +appender <- logger::log_appender() +layout <- logger::log_layout() +on.exit({ + ## reset logger settings + logger::log_threshold(threshold) + logger::log_layout(layout) + logger::log_appender(appender) +}) + +logger::log_appender(logger::appender_stdout) +logger::log_threshold(logger::FATAL) +logger::log_layout(logger::layout_simple) + +test_that("No inheritence", { + params <- list( + foo = 1L, + string = "simple params" + ) + results <- inherit_params(params) + expect_identical(results, params) +}) + +test_that("Simple inheritence works", { + params <- list( + foo = 1L, + string = "simple params", + inherit = "test01" + ) + param_dir <- withr::local_tempdir() + writeLines( + '{ + "inherited_key": 2, + "some_other_key": "test01", + "string": "we should not see this" + }', + file.path(param_dir, "test01.json") + ) + results <- inherit_params( + params = params, + inheritence_search_paths = param_dir + ) + expect_identical( + object = results, + expected = list( + inherited_key = 2L, + some_other_key = "test01", + string = "simple params", + foo = 1L + ) + ) +}) + +test_that("Only inheritence works", { + params <- list( + inherit = "test01" + ) + param_dir <- withr::local_tempdir() + writeLines( + '{ + "inherited_key": 2, + "some_other_key": "test01", + "string": "we should not see this" + }', + file.path(param_dir, "test01.json") + ) + results <- inherit_params( + params = params, + inheritence_search_paths = param_dir + ) + expect_identical( + object = results, + expected = list( + inherited_key = 2L, + some_other_key = "test01", + string = "we should not see this" + ) + ) +}) + +test_that("Simple inheritence picks the correct file", { + params <- list( + foo = 1L, + string = "simple params", + inherit = "test02" + ) + param_dir <- withr::local_tempdir() + writeLines( + '{ + "inherited_key": 2, + "some_other_key": "test01", + "string": "we should not see this" + }', + file.path(param_dir, "test01.json") + ) + writeLines( + '{ + "inherited_key": 3, + "some_other_key": "test02", + "string": "we should not see this either" + }', + file.path(param_dir, "test02.json") + ) + # Note that we're inheriting from test02.json + results <- inherit_params( + params = params, + inheritence_search_paths = param_dir + ) + expect_identical( + object = results, + expected = list( + inherited_key = 3L, + some_other_key = "test02", + string = "simple params", + foo = 1L + ) + ) +}) + +test_that("Nested inheritence works", { + params <- list( + foo = 1L, + string = "simple params", + inherit = "test01" + ) + param_dir <- withr::local_tempdir() + writeLines( + '{ + "inherited_key": 2, + "some_other_key": "test01", + "string": "we should not see this", + "test01": true, + "inherit": "test02" + }', + file.path(param_dir, "test01.json") + ) + writeLines( + '{ + "inherited_key": 3, + "some_other_key": "test02", + "string": "we should not see this either", + "test02": true + }', + file.path(param_dir, "test02.json") + ) + results <- inherit_params( + params = params, + inheritence_search_paths = param_dir + ) + expect_identical( + object = results, + expected = list( + inherited_key = 2L, + some_other_key = "test01", + string = "simple params", + test02 = TRUE, + test01 = TRUE, + foo = 1L + ) + ) +}) + +test_that("Missing inheritence file throws error", { + params <- list( + foo = 1L, + string = "simple params", + inherit = "test01" + ) + param_dir <- withr::local_tempdir() + testthat::expect_error( + inherit_params( + params = params, + inheritence_search_paths = param_dir + ), + regexp = "^Inheritence file not found.$" + ) +}) + +test_that("Multiple inherit keys in params throws error", { + params <- list( + foo = 1L, + string = "simple params", + inherit = "test01", + inherit = "test02" # nolint: duplicate_argument_linter + ) + param_dir <- withr::local_tempdir() + writeLines( + '{ + "inherited_key": 2, + "some_other_key": "test01", + "string": "we should not see this" + }', + file.path(param_dir, "test01.json") + ) + writeLines( + '{ + "inherited_key": 3, + "some_other_key": "test02", + "string": "we should not see this either" + }', + file.path(param_dir, "test02.json") + ) + testthat::expect_error( + inherit_params( + params = params, + inheritence_search_paths = param_dir + ), + regexp = "^Multiple inheritence keys found.$" + ) +}) + +test_that("Multiple values in inherit key throws error", { + params <- list( + foo = 1L, + string = "simple params", + inherit = c("test01", "test02") + ) + param_dir <- withr::local_tempdir() + writeLines( + '{ + "inherited_key": 2, + "some_other_key": "test01", + "string": "we should not see this" + }', + file.path(param_dir, "test01.json") + ) + writeLines( + '{ + "inherited_key": 3, + "some_other_key": "test02", + "string": "we should not see this either" + }', + file.path(param_dir, "test02.json") + ) + testthat::expect_error( + inherit_params( + params = params, + inheritence_search_paths = param_dir + ), + regexp = "^Multiple values in inherit key.$" + ) +}) + +test_that("Circular inheritence throws error", { + params <- list( + foo = 1L, + string = "simple params", + inherit = "test01" + ) + param_dir <- withr::local_tempdir() + writeLines( + '{ + "inherited_key": 2, + "some_other_key": "test01", + "string": "we should not see this", + "inherit": "test02" + }', + file.path(param_dir, "test01.json") + ) + writeLines( + '{ + "inherited_key": 3, + "some_other_key": "test02", + "string": "we should not see this either", + "inherit": "test01" + }', + file.path(param_dir, "test02.json") + ) + testthat::expect_error( + inherit_params( + params = params, + inheritence_search_paths = param_dir + ), + regexp = "^Inheritence loop detected.$" + ) +}) + +test_that("Searching across multiple directories works", { + params <- list( + foo = 1L, + string = "simple params", + inherit = "test01" + ) + first_dir <- withr::local_tempdir() + writeLines( + '{ + "inherited_key": 2, + "dir": "first", + "some_other_key": "test01", + "string": "we should not see this", + "test01": true, + "inherit": "test02" + }', + file.path(first_dir, "test01.json") + ) + second_dir <- withr::local_tempdir() + writeLines( + '{ + "inherited_key": 3, + "dir": "second", + "some_other_key": "test02", + "string": "we should not see this either", + "test02": true + }', + file.path(second_dir, "test02.json") + ) + results <- inherit_params( + params = params, + inheritence_search_paths = c(first_dir, second_dir) + ) + expect_identical( + object = results, + expected = list( + inherited_key = 2L, + dir = "first", # inheriting from first_dir/test01.json + some_other_key = "test01", + string = "simple params", + test02 = TRUE, + test01 = TRUE, + foo = 1L + ) + ) +}) + + +test_that("Searching across multiple directories works", { + params <- list( + foo = 1L, + string = "simple params", + inherit = "test01" + ) + first_dir <- withr::local_tempdir() + writeLines( + '{ + "inherited_key": 2, + "dir": "first", + "some_other_key": "test01", + "string": "we should not see this", + "test01": true + }', + file.path(first_dir, "test01.json") + ) + second_dir <- withr::local_tempdir() + writeLines( + '{ + "inherited_key": 3, + "dir": "second", + "some_other_key": "secret bonus file", + "string": "we should not see this either", + "test02": true + }', + file.path(second_dir, "test01.json") + ) + testthat::expect_warning( + { + results <- inherit_params( + params = params, + inheritence_search_paths = c(first_dir, second_dir) + ) + }, + regexp = "^Multiple inheritence files found.$" + ) + expect_identical( + object = results, + expected = list( + inherited_key = 2L, + dir = "first", # inheriting from first_dir/test01.json + some_other_key = "test01", + string = "simple params", + test01 = TRUE, + foo = 1L + ) + ) +}) diff --git a/tests/testthat/test-parse_params.R b/tests/testthat/test-parse_params.R new file mode 100644 index 00000000..93ac9266 --- /dev/null +++ b/tests/testthat/test-parse_params.R @@ -0,0 +1,208 @@ +## save current settings so that we can reset later +threshold <- logger::log_threshold() +appender <- logger::log_appender() +layout <- logger::log_layout() +on.exit({ + ## reset logger settings + logger::log_threshold(threshold) + logger::log_layout(layout) + logger::log_appender(appender) +}) + +logger::log_appender(logger::appender_stdout) +logger::log_threshold(logger::FATAL) +logger::log_layout(logger::layout_simple) + + +test_that("No inheritence, pass as string", { + json_string <- '{ + "id": 1, + "name": "A green door", + "price": 12.50, + "tags": ["home", "green"] + }' + results <- parse_params(json_string) + expect_identical( + object = results, + expected = list( + id = 1L, + name = "A green door", + price = 12.5, + tags = c("home", "green") + ) + ) +}) + +test_that("No inheritence, pass as file", { + json_string <- '{ + "id": 1, + "name": "A green door", + "price": 12.50, + "tags": ["home", "green"] + }' + json_file <- withr::local_tempfile(fileext = ".json") + writeLines(json_string, json_file) + results <- parse_params(json_file) + expect_identical( + object = results, + expected = list( + id = 1L, + name = "A green door", + price = 12.5, + tags = c("home", "green") + ) + ) +}) + +base_params_dir <- withr::local_tempdir() +base_01_string <- '{ + "name": "A green door", + "tags": ["home", "green"], + "supplier": "ACME Doors" + }' +writeLines( + base_01_string, + file.path(base_params_dir, "base01.json") +) + +test_that("Simple inheritence, pass as string", { + json_string <- '{ + "id": 1, + "price": 12.50, + "inherit": "base01" + }' + results <- parse_params( + json = json_string, + inheritence_search_paths = base_params_dir + ) + expect_identical( + object = results, + expected = list( + name = "A green door", + tags = c("home", "green"), + supplier = "ACME Doors", + id = 1L, + price = 12.5 + ) + ) +}) + +test_that("Simple inheritence, pass as file", { + json_string <- '{ + "id": 1, + "price": 12.50, + "inherit": "base01" + }' + json_file <- withr::local_tempfile(fileext = ".json") + writeLines(json_string, json_file) + results <- parse_params( + json = json_file, + inheritence_search_paths = base_params_dir + ) + expect_identical( + object = results, + expected = list( + name = "A green door", + tags = c("home", "green"), + supplier = "ACME Doors", + id = 1L, + price = 12.5 + ) + ) +}) + +product_schema <- '{ + "$schema": "http://json-schema.org/draft-04/schema#", + "title": "Product", + "description": "A product from Acme\'s catalog", + "type": "object", + "properties": { + "id": { + "description": "The unique identifier for a product", + "type": "integer" + }, + "name": { + "description": "Name of the product", + "type": "string" + }, + "price": { + "type": "number", + "minimum": 0, + "exclusiveMinimum": true + }, + "tags": { + "type": "array", + "items": { + "type": "string" + }, + "minItems": 1, + "uniqueItems": true + } + }, + "required": ["id", "name", "price"] +}' +schema_dir <- withr::local_tempdir() +schema_file <- file.path(schema_dir, "product.json") +writeLines(product_schema, schema_file) + +test_that("No inheritence, pass as string, validation works", { + json_string <- '{ + "id": 1, + "name": "A green door", + "price": 12.50, + "tags": ["home", "green"] + }' + results <- parse_params( + json = json_string, + schema_file = schema_file + ) + expect_identical( + object = results, + expected = list( + id = 1L, + name = "A green door", + price = 12.5, + tags = c("home", "green") + ) + ) +}) + +test_that("No inheritence, pass as string, failing validation works", { + json_string <- '{ + "id": 1.5, + "price": 12.50, + "tags": ["home", "green"] + }' + testthat::expect_error( + object = { + parse_params( + json = json_string, + schema_file = schema_file + ) + }, + regexp = "^JSON Validation failed.$" + ) +}) + +test_that("simple inheritence, pass as string, validation works", { + json_string <- '{ + "id": 1, + "price": 12.50, + "inherit": "base01" + }' + results <- parse_params( + json = json_string, + inheritence_search_paths = base_params_dir, + schema_file = schema_file + ) + expect_identical( + object = results, + expected = list( + name = "A green door", + tags = c("home", "green"), + supplier = "ACME Doors", + id = 1L, + price = 12.5 + ) + ) +})