Skip to content

Commit

Permalink
Merge pull request #1 from DrTTrousers/develop(ru)
Browse files Browse the repository at this point in the history
Fixes snowflake duplication error
  • Loading branch information
DrTTrousers authored May 3, 2023
2 parents 0cf869d + 0846de6 commit 967a912
Show file tree
Hide file tree
Showing 14 changed files with 831 additions and 132 deletions.
2 changes: 2 additions & 0 deletions DESCRIPTION
Original file line number Diff line number Diff line change
Expand Up @@ -30,6 +30,7 @@ Imports:
digest,
methods,
tibble,
ResultModelManager (>= 0.3.0),
SqlRender (>= 1.11.0)
Suggests:
testthat (>= 3.0.0),
Expand All @@ -40,6 +41,7 @@ Suggests:
withr
Remotes:
ohdsi/CohortGenerator,
ohdsi/ResultModelManager,
ohdsi/Eunomia
VignetteBuilder: knitr
NeedsCompilation: no
Expand Down
1 change: 1 addition & 0 deletions NAMESPACE
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@ export(addModuleSpecifications)
export(addSharedResources)
export(createCdmExecutionSettings)
export(createEmptyAnalysisSpecificiations)
export(createResultDataModels)
export(createResultsExecutionSettings)
export(ensureAllModulesInstantiated)
export(execute)
Expand Down
129 changes: 76 additions & 53 deletions R/Execution.R
Original file line number Diff line number Diff line change
Expand Up @@ -74,6 +74,7 @@ execute <- function(analysisSpecifications,
}
dependencies <- extractDependencies(modules)


fileName <- generateTargetsScript(
analysisSpecifications = analysisSpecifications,
executionSettings = executionSettings,
Expand All @@ -82,7 +83,6 @@ execute <- function(analysisSpecifications,
restart = restart,
keyringName = keyringName
)

# targets::tar_manifest(script = fileName)
# targets::tar_glimpse(script = fileName)
targets::tar_make(script = fileName, store = file.path(executionScriptFolder, "_targets"))
Expand All @@ -93,33 +93,67 @@ generateTargetsScript <- function(analysisSpecifications, executionSettings, dep
if (restart) {
return(fileName)
}
# Store settings objects in the temp folder so they are available in targets

### Note anything inisde this block will be scoped inside the targets script file
targets::tar_script({
##
# Generated by Strategus - not advisable to edit by hand
##
analysisSpecificationsLoad <- readRDS(analysisSpecificationsFileName)
moduleToTargetNames <- readRDS(moduleToTargetNamesFileName)
dependencies <- readRDS(dependenciesFileName)

library(dplyr)
tar_option_set(packages = c('Strategus', 'keyring'), imports = c('Strategus', 'keyring'))
targetList <- list(
tar_target(analysisSpecifications, readRDS(analysisSpecificationsFileName)),
# NOTE Execution settings could be mapped to many different cdms making re-execution across cdms much simpler
tar_target(executionSettings, readRDS(executionSettingsFileName)),
tar_target(keyringSettings, readRDS(keyringSettingsFileName))
)

# factory for producing module targets based on their dependencies
# This could be inside Strategus as an exported function
# it would also be much cleaner to use a targets pattern = cross(analysisSpecifications$moduleSpecifications)
# however, working out how to handle dependencies wasn't obvious
# This approach could be modified to allow multiple executionSettings, but that would require a substantial re-write
for (i in 1:length(analysisSpecificationsLoad$moduleSpecifications)) {
moduleSpecification <- analysisSpecificationsLoad$moduleSpecifications[[i]]
targetName <- sprintf("%s_%d", moduleSpecification$module, i)
dependencyModules <- dependencies %>%
filter(.data$module == moduleSpecification$module) %>%
pull(.data$dependsOn)

dependencyTargetNames <- moduleToTargetNames %>%
filter(.data$module %in% dependencyModules) %>%
pull(.data$targetName)

# Use of tar_target_raw allows dynamic names
targetList[[length(targetList) + 1]] <- tar_target_raw(targetName,
substitute(Strategus:::runModule(analysisSpecifications, keyringSettings, i, executionSettings),
env = list(i = i)),
deps = c("analysisSpecifications", "keyringSettings", "executionSettings", dependencyTargetNames))

if (execResultsUpload) {
resultsTargetName <- paste0(targetName, "_results_upload")
targetList[[length(targetList) + 1]] <- tar_target_raw(resultsTargetName,
substitute(Strategus:::runResultsUpload(analysisSpecifications, keyringSettings, i, executionSettings),
env = list(i = i)),
deps = c("analysisSpecifications", "keyringSettings", "executionSettings", targetName))
}
}

targetList
}, script = fileName)

#Store settings objects in the temp folder so they are available in targets
analysisSpecificationsFileName <- gsub("\\\\", "/", file.path(executionScriptFolder, "analysisSpecifications.rds"))
saveRDS(analysisSpecifications, analysisSpecificationsFileName)
executionSettingsFileName <- gsub("\\\\", "/", file.path(executionScriptFolder, "executionSettings.rds"))
saveRDS(executionSettings, executionSettingsFileName)
keyringSettingsFileName <- gsub("\\\\", "/", file.path(executionScriptFolder, "keyringSettings.rds"))
saveRDS(list(keyringName = keyringName), keyringSettingsFileName)


# Dynamically generate targets script based on analysis specifications
lines <- c(
"library(targets)",
"tar_option_set(packages = c('Strategus', 'keyring'))",
"list(",
" tar_target(",
" analysisSpecifications,",
sprintf(" readRDS('%s')", analysisSpecificationsFileName),
" ),",
" tar_target(",
" executionSettings,",
sprintf(" readRDS('%s')", executionSettingsFileName),
" ),",
" tar_target(",
" keyringSettings,",
sprintf(" readRDS('%s')", keyringSettingsFileName),
" ),"
)
# Generate target names by module type
moduleToTargetNames <- list()
for (i in 1:length(analysisSpecifications$moduleSpecifications)) {
Expand All @@ -131,38 +165,27 @@ generateTargetsScript <- function(analysisSpecifications, executionSettings, dep
)
}
moduleToTargetNames <- bind_rows(moduleToTargetNames)
moduleToTargetNamesFileName <- gsub("\\\\", "/", file.path(executionScriptFolder, "moduleTargetNames.rds"))
saveRDS(moduleToTargetNames, moduleToTargetNamesFileName)

dependenciesFileName <- gsub("\\\\", "/", file.path(executionScriptFolder, "dependencies.rds"))
saveRDS(dependencies, dependenciesFileName)

execResultsUpload <- all(c(is(executionSettings, "CdmExecutionSettings"),
!is.null(executionSettings$resultsConnectionDetailsReference),
!is.null(executionSettings$resultsDatabaseSchema)))

# Settings required inside script. There is probably a much cleaner way of doing this
writeLines(c(
sprintf("analysisSpecificationsFileName <- '%s'", analysisSpecificationsFileName),
sprintf("executionSettingsFileName <- '%s'", executionSettingsFileName),
sprintf("keyringSettingsFileName <- '%s'", keyringSettingsFileName),
sprintf("moduleToTargetNamesFileName <- '%s'", moduleToTargetNamesFileName),
sprintf("dependenciesFileName <- '%s'", dependenciesFileName),
sprintf("execResultsUpload <- '%s'", execResultsUpload),
readLines(fileName)
), fileName)

# Generate targets code, inserting dependencies
for (i in 1:length(analysisSpecifications$moduleSpecifications)) {
moduleSpecification <- analysisSpecifications$moduleSpecifications[[i]]
targetName <- sprintf("%s_%d", moduleSpecification$module, i)
dependencyModules <- dependencies %>%
filter(.data$module == moduleSpecification$module) %>%
pull(.data$dependsOn)
dependencyTargetNames <- moduleToTargetNames %>%
filter(.data$module %in% dependencyModules) %>%
pull(.data$targetName)

command <- sprintf(
"Strategus:::runModule(analysisSpecifications, keyringSettings, %d, executionSettings%s)",
i,
ifelse(length(dependencyTargetNames) == 0, "", sprintf(", %s", paste(dependencyTargetNames, collapse = ", ")))
)


lines <- c(
lines,
" tar_target(",
sprintf(" %s,", targetName),
sprintf(" %s", command),
ifelse(i == length(analysisSpecifications$moduleSpecifications), " )", " ),")
)
}

lines <- c(lines, ")")

sink(fileName)
cat(paste(lines, collapse = "\n"))
sink()
return(fileName)
}

86 changes: 86 additions & 0 deletions R/ModuleEnv.R
Original file line number Diff line number Diff line change
@@ -0,0 +1,86 @@
# Copyright 2023 Observational Health Data Sciences and Informatics
#
# This file is part of Strategus
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

.handleInjectVar <- function (x) {
hVar <- function(x) {
if (is.character(x)) {
return(sprintf('"%s"', x))
} else {
return(sprintf('%s', x))
}
}

if (length(x) == 1) {
return(hVar(x))
} else if (is.vector(x)) {
innerVars <- hVar(x)
return(paste0("c(", paste(innerVars, collapse = ", "), ")"))
} else {
stop("cannot handle complex data structures in variable injection")
}
}

#' Load module execution space inside and renv
#' inspired by targets::tar_script but allowing custom variable execution
#'
#' Designed to allow more human readable code that is executed inside a module as well as simple variable substituion
#' for injecting constants (e.g. simple parameters or file paths used inside and outside of modules)
#'
#' This pattern also allows dependency injection which could be used if you don't want to use and renv and (instead)
#' would like to use docker images or just execution in the base environment for testing/debugging
#'
#' @param code code block to execute
#' @param moduleFolder Instantiated Strategus module folder
#' @param injectVars list of var names list(name=value) to replace (e.g. replace list(foo = "some string") will
#' find the pattern foo and replace it with the string some string - be careful!
#' @param tempScriptFile tempFile to write script to (ret
#' @param job run as rstudio job
#' @param processName String name for process
#' @returns NULL invisibly
withModuleRenv <- function(code,
moduleFolder,
injectVars = list(),
tempScriptFile = tempfile(fileext = ".R"),
useLocalStrategusLibrary = TRUE,
job = FALSE,
processName = paste(moduleFolder, "_renv_run")) {
# convert human readable code to a string for writing
script <- as.character(substitute(code))[-1]
# Insert variables
for (name in names(injectVars)) {
rep <- .handleInjectVar(injectVars[[name]])
script <- gsub(name, rep, script)
}

# Enforce attachment of Strategus from calling process - note one inside the renv
if (useLocalStrategusLibrary) {
libPath <- file.path(find.package("Strategus"), "../")
script <- c(sprintf("library(Strategus, lib.loc = '%s')", libPath),
script)
}

# Write file and execute script inside an renv
fileConn <- file(tempScriptFile)
writeLines(script, fileConn)
close(fileConn)
renv::run(
script = tempScriptFile,
job = job,
name = processName,
project = moduleFolder
)
return(invisible(NULL))
}
8 changes: 4 additions & 4 deletions R/ModuleInstantiation.R
Original file line number Diff line number Diff line change
Expand Up @@ -223,10 +223,10 @@ getModuleRenvDependencies <- function(moduleFolder) {
)

missingFiles <- tibble::enframe(renvRequiredFiles) %>%
mutate(fileExists = file.exists(file.path(moduleFolder, .data$value))) %>%
rename(fileName = .data$value) %>%
select(.data$fileName, .data$fileExists) %>%
filter(.data$fileExists == FALSE)
dplyr::mutate(fileExists = file.exists(file.path(moduleFolder, .data$value))) %>%
dplyr::rename(fileName = .data$value) %>%
dplyr::select("fileName", "fileExists") %>%
dplyr::filter(.data$fileExists == FALSE)

invisible(missingFiles)
}
Expand Down
Loading

0 comments on commit 967a912

Please sign in to comment.