diff --git a/.gitignore b/.gitignore index 39ae5d3..7924745 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,5 @@ output/*.sql +output/*.txt .DS_Store .Rproj.user errorReport.txt diff --git a/DESCRIPTION b/DESCRIPTION index 620d0bb..90f2334 100644 --- a/DESCRIPTION +++ b/DESCRIPTION @@ -1,7 +1,7 @@ Package: ETLSyntheaBuilder Type: Package Title: A Builder for Converting the Synthea Data to the OMOP CDM -Version: 2.0 +Version: 2.1 mDate: 2021-12-04 Author: Anthony Molinaro [aut, cre], Clair Blacketer [aut], @@ -28,7 +28,7 @@ Remotes: OHDSI/CommonDataModel Encoding: UTF-8 LazyData: true -RoxygenNote: 7.2.3 +RoxygenNote: 7.3.1 Suggests: rmarkdown, knitr, diff --git a/R/CreateMapAndRollupTables.r b/R/CreateMapAndRollupTables.r index 5bf6261..a7c61fb 100644 --- a/R/CreateMapAndRollupTables.r +++ b/R/CreateMapAndRollupTables.r @@ -19,7 +19,7 @@ #' so for example 'cdm_instance.dbo'. #' @param cdmVersion The version of your CDM. Currently "5.3" and "5.4". #' @param syntheaVersion The version of Synthea used to generate the csv files. -#' Currently "2.7.0", "3.0.0", "3.1.0" and "3.2.0" are supported. +#' Currently "2.7.0", "3.0.0", "3.1.0", "3.2.0" and "3.3.0" are supported. #' @param cdmSourceName The source name to insert into the CDM_SOURCE table. Default is Synthea synthetic health database. #' @param cdmSourceAbbreviation The source abbreviation to insert into the CDM_SOURCE table. Default is Synthea. #' @param cdmHolder The holder to insert into the CDM_SOURCE table. Default is OHDSI @@ -46,11 +46,11 @@ CreateMapAndRollupTables <- function(connectionDetails, stop("Unsupported CDM specified. Supported CDM versions are \"5.3\" and \"5.4\".") } - supportedSyntheaVersions <- c("2.7.0", "3.0.0", "3.1.0", "3.2.0") + supportedSyntheaVersions <- c("2.7.0", "3.0.0", "3.1.0", "3.2.0", "3.3.0") if (!(syntheaVersion %in% supportedSyntheaVersions)) stop( - "Invalid Synthea version specified. Currently \"2.7.0\", \"3.0.0\",\"3.1.0\", and \"3.2.0\" are supported." + "Invalid Synthea version specified. Currently \"2.7.0\", \"3.0.0\",\"3.1.0\", \"3.2.0\", and \"3.3.0\" are supported." ) # Create Vocabulary mapping tables diff --git a/R/CreateSyntheaTables.r b/R/CreateSyntheaTables.r index 43a8954..70ce8be 100644 --- a/R/CreateSyntheaTables.r +++ b/R/CreateSyntheaTables.r @@ -10,7 +10,7 @@ #' Server, this should specify both the database and the schema, #' so for example 'cdm_instance.dbo'. #' @param syntheaVersion The version of Synthea used to generate the csv files. -#' Currently "2.7.0", "3.0.0", "3.1.0", and "3.2.0" are is supported. +#' Currently "2.7.0", "3.0.0", "3.1.0", "3.2.0" and "3.3.0" are is supported. #' #'@export @@ -29,9 +29,11 @@ CreateSyntheaTables <- sqlFilePath <- "synthea_version/v310" else if (syntheaVersion == "3.2.0") sqlFilePath <- "synthea_version/v320" + else if (syntheaVersion == "3.3.0") + sqlFilePath <- "synthea_version/v330" else stop( - "Invalid synthea version specified. Currently \"2.7.0\", \"3.0.0\", \"3.1.0\" and \"3.2.0\" are supported." + "Invalid synthea version specified. Currently \"2.7.0\", \"3.0.0\", \"3.1.0\" , \"3.2.0\" and \"3.3.0\" are supported." ) sqlFilename <- diff --git a/R/LoadEventTables.r b/R/LoadEventTables.r index 9ffe996..d1e1e48 100644 --- a/R/LoadEventTables.r +++ b/R/LoadEventTables.r @@ -18,7 +18,7 @@ #' so for example 'cdm_instance.dbo'. #' @param cdmVersion The version of your CDM. Currently "5.3" and "5.4". #' @param syntheaVersion The version of Synthea used to generate the csv files. -#' Currently "2.7.0","3.0.0","3.1.0" and "3.2.0" are supported. +#' Currently "2.7.0","3.0.0","3.1.0","3.2.0" and "3.3.0" are supported. #' @param cdmSourceName The source name to insert into the CDM_SOURCE table. Default is Synthea synthetic health database. #' @param cdmSourceAbbreviation The source abbreviation to insert into the CDM_SOURCE table. Default is Synthea. #' @param cdmHolder The holder to insert into the CDM_SOURCE table. Default is OHDSI @@ -51,11 +51,11 @@ LoadEventTables <- function(connectionDetails, stop("Unsupported CDM specified. Supported CDM versions are \"5.3\" and \"5.4\".") } - supportedSyntheaVersions <- c("2.7.0", "3.0.0", "3.1.0", "3.2.0") + supportedSyntheaVersions <- c("2.7.0", "3.0.0", "3.1.0", "3.2.0", "3.3.0") if (!(syntheaVersion %in% supportedSyntheaVersions)) stop( - "Invalid Synthea version specified. Currently \"2.7.0\", \"3.0.0\",\"3.1.0\", and \"3.2.0\" are supported." + "Invalid Synthea version specified. Currently \"2.7.0\", \"3.0.0\",\"3.1.0\",\"3.2.0\", and \"3.3.0\" are supported." ) if (createIndices) { @@ -177,7 +177,8 @@ LoadEventTables <- function(connectionDetails, packageName = "ETLSyntheaBuilder", dbms = connectionDetails$dbms, cdm_schema = cdmSchema, - synthea_schema = syntheaSchema + synthea_schema = syntheaSchema, + synthea_version = syntheaVersion ) runStep(sql, fileQuery) @@ -299,7 +300,7 @@ LoadEventTables <- function(connectionDetails, # cost if (syntheaVersion == "2.7.0") fileQuery <- "insert_cost_v270.sql" - else if (syntheaVersion %in% c("3.0.0", "3.1.0", "3.2.0")) + else if (syntheaVersion %in% c("3.0.0", "3.1.0", "3.2.0", "3.3.0")) fileQuery <- "insert_cost_v300.sql" sql <- SqlRender::loadRenderTranslateSql( diff --git a/R/LoadVocabFromCsv.r b/R/LoadVocabFromCsv.r index c2b6bf2..336149e 100644 --- a/R/LoadVocabFromCsv.r +++ b/R/LoadVocabFromCsv.r @@ -9,10 +9,11 @@ #' \code{DatabaseConnector} package. #' @param cdmSchema The name of the database schema that will contain the Vocabulary (and CDM) #' tables. Requires read and write permissions to this database. On SQL -#' Server, this should specifiy both the database and the schema, +#' Server, this should specify both the database and the schema, #' so for example 'cdm_instance.dbo'. #' @param vocabFileLoc The location of the vocabulary csv files. #' @param bulkLoad Boolean flag indicating whether or not to use bulk loading (if possible). Default is FALSE. +#' @param delimiter Parameter to specify the delimiter of the csv file. By default the function expects the file to be tab delimited ("\\t") based on the export from Athena. #' #'@export @@ -21,7 +22,8 @@ LoadVocabFromCsv <- function(connectionDetails, cdmSchema, vocabFileLoc, - bulkLoad = FALSE) + bulkLoad = FALSE, + delimiter = "\t") { csvList <- c( "concept.csv", @@ -50,7 +52,7 @@ LoadVocabFromCsv <- file = paste0(vocabFileLoc, "/", csv), stringsAsFactors = FALSE, header = TRUE, - sep = "\t", + sep = delimiter, na.strings = "" ) diff --git a/R/createExtraIndices.R b/R/createExtraIndices.R index 9d435d8..7f97933 100644 --- a/R/createExtraIndices.R +++ b/R/createExtraIndices.R @@ -8,13 +8,13 @@ #' Server, this should specify both the database and the schema, so for example 'cdm_instance.dbo'. #' @param syntheaSchema The name of the Synthea database schema. Requires read and write permissions to this schema. On SQL #' Server, this should specify both the database and the schema, so for example 'synthea.dbo'. -#' @param syntheaVersion Your Synthea version. Currently "2.7.0", "3.0.0", "3.1.0" and "3.2.0" are supported. +#' @param syntheaVersion Your Synthea version. Currently "2.7.0", "3.0.0", "3.1.0", "3.2.0" and "3.3.0" are supported. #' @param outputFolder Location of the SQL scripts if sqlOnly = TRUE. Default is NULL. #' @param sqlOnly A boolean that determines whether to create the indices or generate a SQL scripts. Default is FALSE. #' #' @details This function creates indices which have been found to speed up certain long-running INSERT queries in LoadEventTables, #' for some users. Indices are created on the intermediate vocabulary mapping tables; the person & provider CDM tables; -#' and the claims_transactions Synthea table (in Synthea 3.0.0, 3.1.0 and 3.2.0) . +#' and the claims_transactions Synthea table (in Synthea 3.0.0, 3.1.0, 3.2.0 and 3.3.0) . #' #' @importFrom utils head #' diff --git a/README.md b/README.md index 0ceb7d2..f99ca67 100644 --- a/README.md +++ b/README.md @@ -16,7 +16,7 @@ Follow the steps on the [synthea wiki](https://github.com/synthetichealth/synthe # Valid CDM versions are determined by executing CommonDataModel::listSupportedVersions(). # The strings representing supported CDM versions are currently "5.3" and "5.4". # The Synthea version we use in this example is 2.7.0. - # However, at this time we also support 3.0.0, 3.1.0 and 3.2.0. + # However, at this time we also support 3.0.0, 3.1.0, 3.2.0 and 3.3.0. # Please note that Synthea's MASTER branch is always active and this package will be updated to support # future versions as possible. # The schema to load the Synthea tables is called "native". diff --git a/docs/404.html b/docs/404.html index 96ef43e..2f4ea89 100644 --- a/docs/404.html +++ b/docs/404.html @@ -32,7 +32,7 @@ ETLSyntheaBuilder - 2.0 + 2.1 diff --git a/docs/articles/care_site.html b/docs/articles/care_site.html index b24e257..52ef6d2 100644 --- a/docs/articles/care_site.html +++ b/docs/articles/care_site.html @@ -33,7 +33,7 @@ ETLSyntheaBuilder - 2.0 + 2.1 diff --git a/docs/articles/condition_occurrence.html b/docs/articles/condition_occurrence.html index 4307e2c..f7dc2b2 100644 --- a/docs/articles/condition_occurrence.html +++ b/docs/articles/condition_occurrence.html @@ -33,7 +33,7 @@ ETLSyntheaBuilder - 2.0 + 2.1 diff --git a/docs/articles/cost.html b/docs/articles/cost.html index 1ac4721..08c7c86 100644 --- a/docs/articles/cost.html +++ b/docs/articles/cost.html @@ -33,7 +33,7 @@ ETLSyntheaBuilder - 2.0 + 2.1 diff --git a/docs/articles/death.html b/docs/articles/death.html index 01e4b4b..379c38c 100644 --- a/docs/articles/death.html +++ b/docs/articles/death.html @@ -33,7 +33,7 @@ ETLSyntheaBuilder - 2.0 + 2.1 diff --git a/docs/articles/device_exposure.html b/docs/articles/device_exposure.html index e5ca65c..4b3d2db 100644 --- a/docs/articles/device_exposure.html +++ b/docs/articles/device_exposure.html @@ -33,7 +33,7 @@ ETLSyntheaBuilder - 2.0 + 2.1 diff --git a/docs/articles/drug_exposure.html b/docs/articles/drug_exposure.html index 344aa3d..705f875 100644 --- a/docs/articles/drug_exposure.html +++ b/docs/articles/drug_exposure.html @@ -33,7 +33,7 @@ ETLSyntheaBuilder - 2.0 + 2.1 diff --git a/docs/articles/index.html b/docs/articles/index.html index 27b7c10..e82a198 100644 --- a/docs/articles/index.html +++ b/docs/articles/index.html @@ -17,7 +17,7 @@ ETLSyntheaBuilder - 2.0 + 2.1 diff --git a/docs/articles/location.html b/docs/articles/location.html index 44edb8d..853f63d 100644 --- a/docs/articles/location.html +++ b/docs/articles/location.html @@ -33,7 +33,7 @@ ETLSyntheaBuilder - 2.0 + 2.1 diff --git a/docs/articles/measurement.html b/docs/articles/measurement.html index 8f4d343..9ea8823 100644 --- a/docs/articles/measurement.html +++ b/docs/articles/measurement.html @@ -33,7 +33,7 @@ ETLSyntheaBuilder - 2.0 + 2.1 diff --git a/docs/articles/observation.html b/docs/articles/observation.html index 077062b..5009c61 100644 --- a/docs/articles/observation.html +++ b/docs/articles/observation.html @@ -33,7 +33,7 @@ ETLSyntheaBuilder - 2.0 + 2.1 diff --git a/docs/articles/observation_period.html b/docs/articles/observation_period.html index 3744d30..9f24ca4 100644 --- a/docs/articles/observation_period.html +++ b/docs/articles/observation_period.html @@ -33,7 +33,7 @@ ETLSyntheaBuilder - 2.0 + 2.1 diff --git a/docs/articles/payer_plan_period.html b/docs/articles/payer_plan_period.html index 6d9dcd4..c022d3a 100644 --- a/docs/articles/payer_plan_period.html +++ b/docs/articles/payer_plan_period.html @@ -33,7 +33,7 @@ ETLSyntheaBuilder - 2.0 + 2.1 diff --git a/docs/articles/person.html b/docs/articles/person.html index 8fe3e17..6b30f04 100644 --- a/docs/articles/person.html +++ b/docs/articles/person.html @@ -33,7 +33,7 @@ ETLSyntheaBuilder - 2.0 + 2.1 diff --git a/docs/articles/procedure_occurrence.html b/docs/articles/procedure_occurrence.html index 7940f07..38c31f8 100644 --- a/docs/articles/procedure_occurrence.html +++ b/docs/articles/procedure_occurrence.html @@ -33,7 +33,7 @@ ETLSyntheaBuilder - 2.0 + 2.1 diff --git a/docs/articles/provider.html b/docs/articles/provider.html index c7b2060..8a1e8ce 100644 --- a/docs/articles/provider.html +++ b/docs/articles/provider.html @@ -33,7 +33,7 @@ ETLSyntheaBuilder - 2.0 + 2.1 diff --git a/docs/articles/visit_detail.html b/docs/articles/visit_detail.html index 5cfb328..3848427 100644 --- a/docs/articles/visit_detail.html +++ b/docs/articles/visit_detail.html @@ -33,7 +33,7 @@ ETLSyntheaBuilder - 2.0 + 2.1 diff --git a/docs/articles/visit_occurrence.html b/docs/articles/visit_occurrence.html index 18a8bdb..483c0ad 100644 --- a/docs/articles/visit_occurrence.html +++ b/docs/articles/visit_occurrence.html @@ -33,7 +33,7 @@ ETLSyntheaBuilder - 2.0 + 2.1 diff --git a/docs/authors.html b/docs/authors.html index a05143c..e4666eb 100644 --- a/docs/authors.html +++ b/docs/authors.html @@ -17,7 +17,7 @@ ETLSyntheaBuilder - 2.0 + 2.1 @@ -109,13 +109,13 @@

Citation

Molinaro A, Blacketer C, DeFalco F, Burrows E (2024). ETLSyntheaBuilder: A Builder for Converting the Synthea Data to the OMOP CDM. -R package version 2.0. +R package version 2.1. 

@Manual{,
   title = {ETLSyntheaBuilder: A Builder for Converting the Synthea Data to the OMOP CDM},
   author = {Anthony Molinaro and Clair Blacketer and Frank DeFalco and Evanette Burrows},
   year = {2024},
-  note = {R package version 2.0},
+  note = {R package version 2.1},
 }
diff --git a/docs/index.html b/docs/index.html index 44768c3..68c3ee7 100644 --- a/docs/index.html +++ b/docs/index.html @@ -39,7 +39,7 @@ ETLSyntheaBuilder - 2.0 + 2.1 @@ -138,7 +138,7 @@

Step by Step Example# Valid CDM versions are determined by executing CommonDataModel::listSupportedVersions(). # The strings representing supported CDM versions are currently "5.3" and "5.4". # The Synthea version we use in this example is 2.7.0. - # However, at this time we also support 3.0.0, 3.1.0 and 3.2.0. + # However, at this time we also support 3.0.0, 3.1.0, 3.2.0 and 3.3.0. # Please note that Synthea's MASTER branch is always active and this package will be updated to support # future versions as possible. # The schema to load the Synthea tables is called "native". @@ -175,7 +175,7 @@

Step by Step ExampleETLSyntheaBuilder::CreateMapAndRollupTables(connectionDetails = cd, cdmSchema = cdmSchema, syntheaSchema = syntheaSchema, cdmVersion = cdmVersion, syntheaVersion = syntheaVersion) ## Optional Step to create extra indices -ETLSyntheaBuilder::CreateExtraIndices(connectionDetails = cd, cdmSchema = cdmSchema, syntheaSchema = syntheaSchema, syntheaVersion = syntheaVersion) +ETLSyntheaBuilder::CreateExtraIndices(connectionDetails = cd, cdmSchema = cdmSchema, syntheaSchema = syntheaSchema, syntheaVersion = syntheaVersion) ETLSyntheaBuilder::LoadEventTables(connectionDetails = cd, cdmSchema = cdmSchema, syntheaSchema = syntheaSchema, cdmVersion = cdmVersion, syntheaVersion = syntheaVersion) diff --git a/docs/pkgdown.yml b/docs/pkgdown.yml index 70c7c01..9139116 100644 --- a/docs/pkgdown.yml +++ b/docs/pkgdown.yml @@ -18,5 +18,5 @@ articles: provider: provider.html visit_detail: visit_detail.html visit_occurrence: visit_occurrence.html -last_built: 2024-06-02T09:56Z +last_built: 2024-10-04T17:52Z diff --git a/docs/reference/CreateCDMIndexAndConstraintScripts.html b/docs/reference/CreateCDMIndexAndConstraintScripts.html index d0304b5..e9a9561 100644 --- a/docs/reference/CreateCDMIndexAndConstraintScripts.html +++ b/docs/reference/CreateCDMIndexAndConstraintScripts.html @@ -17,7 +17,7 @@ ETLSyntheaBuilder - 2.0 + 2.1 diff --git a/docs/reference/CreateCDMIndices.html b/docs/reference/CreateCDMIndices.html index 477df74..b5c57ce 100644 --- a/docs/reference/CreateCDMIndices.html +++ b/docs/reference/CreateCDMIndices.html @@ -17,7 +17,7 @@ ETLSyntheaBuilder - 2.0 + 2.1 diff --git a/docs/reference/CreateCDMTables.html b/docs/reference/CreateCDMTables.html index e623e12..1d072c2 100644 --- a/docs/reference/CreateCDMTables.html +++ b/docs/reference/CreateCDMTables.html @@ -17,7 +17,7 @@ ETLSyntheaBuilder - 2.0 + 2.1 diff --git a/docs/reference/CreateMapAndRollupTables.html b/docs/reference/CreateMapAndRollupTables.html index 5ad159e..21c3de2 100644 --- a/docs/reference/CreateMapAndRollupTables.html +++ b/docs/reference/CreateMapAndRollupTables.html @@ -18,7 +18,7 @@ ETLSyntheaBuilder - 2.0 + 2.1 @@ -144,7 +144,7 @@

Arguments

syntheaVersion

The version of Synthea used to generate the csv files. -Currently "2.7.0", "3.0.0", "3.1.0" and "3.2.0" are supported.

+Currently "2.7.0", "3.0.0", "3.1.0", "3.2.0" and "3.3.0" are supported.

cdmSourceName
diff --git a/docs/reference/CreateSyntheaTables.html b/docs/reference/CreateSyntheaTables.html index ee8ff77..f35e7e0 100644 --- a/docs/reference/CreateSyntheaTables.html +++ b/docs/reference/CreateSyntheaTables.html @@ -17,7 +17,7 @@ ETLSyntheaBuilder - 2.0 + 2.1 @@ -119,7 +119,7 @@

Arguments

syntheaVersion

The version of Synthea used to generate the csv files. -Currently "2.7.0", "3.0.0", "3.1.0", and "3.2.0" are is supported.

+Currently "2.7.0", "3.0.0", "3.1.0", "3.2.0" and "3.3.0" are is supported.

diff --git a/docs/reference/CreateVisitRollupTables.html b/docs/reference/CreateVisitRollupTables.html index f769921..f056e7e 100644 --- a/docs/reference/CreateVisitRollupTables.html +++ b/docs/reference/CreateVisitRollupTables.html @@ -17,7 +17,7 @@ ETLSyntheaBuilder - 2.0 + 2.1 diff --git a/docs/reference/CreateVocabMapTables.html b/docs/reference/CreateVocabMapTables.html index fc11cc3..ef550b8 100644 --- a/docs/reference/CreateVocabMapTables.html +++ b/docs/reference/CreateVocabMapTables.html @@ -17,7 +17,7 @@ ETLSyntheaBuilder - 2.0 + 2.1 @@ -113,7 +113,7 @@

Arguments

cdmSchema

The name of the database schema that will contain the Vocab mapping tables. Requires read and write permissions to this database. On SQL -Server, this should specifiy both the database and the schema, +Server, this should specify both the database and the schema, so for example 'cdm_instance.dbo'.

diff --git a/docs/reference/DropEventTables.html b/docs/reference/DropEventTables.html index 68cd9ad..e106d3d 100644 --- a/docs/reference/DropEventTables.html +++ b/docs/reference/DropEventTables.html @@ -17,7 +17,7 @@ ETLSyntheaBuilder - 2.0 + 2.1 diff --git a/docs/reference/DropMapAndRollupTables.html b/docs/reference/DropMapAndRollupTables.html index 3ba116f..7b358bb 100644 --- a/docs/reference/DropMapAndRollupTables.html +++ b/docs/reference/DropMapAndRollupTables.html @@ -17,7 +17,7 @@ ETLSyntheaBuilder - 2.0 + 2.1 diff --git a/docs/reference/DropSyntheaTables.html b/docs/reference/DropSyntheaTables.html index f470947..59ebe68 100644 --- a/docs/reference/DropSyntheaTables.html +++ b/docs/reference/DropSyntheaTables.html @@ -17,7 +17,7 @@ ETLSyntheaBuilder - 2.0 + 2.1 diff --git a/docs/reference/DropVocabTables.html b/docs/reference/DropVocabTables.html index 571d280..f6c4720 100644 --- a/docs/reference/DropVocabTables.html +++ b/docs/reference/DropVocabTables.html @@ -17,7 +17,7 @@ ETLSyntheaBuilder - 2.0 + 2.1 diff --git a/docs/reference/ETLSyntheaBuilder-package.html b/docs/reference/ETLSyntheaBuilder-package.html index 131a970..d19be62 100644 --- a/docs/reference/ETLSyntheaBuilder-package.html +++ b/docs/reference/ETLSyntheaBuilder-package.html @@ -17,7 +17,7 @@ ETLSyntheaBuilder - 2.0 + 2.1 diff --git a/docs/reference/LoadEventTables.html b/docs/reference/LoadEventTables.html index 6c9c6a7..2a717fb 100644 --- a/docs/reference/LoadEventTables.html +++ b/docs/reference/LoadEventTables.html @@ -17,7 +17,7 @@ ETLSyntheaBuilder - 2.0 + 2.1 @@ -143,7 +143,7 @@

Arguments

syntheaVersion

The version of Synthea used to generate the csv files. -Currently "2.7.0","3.0.0","3.1.0" and "3.2.0" are supported.

+Currently "2.7.0","3.0.0","3.1.0","3.2.0" and "3.3.0" are supported.

cdmSourceName
diff --git a/docs/reference/LoadSyntheaTables.html b/docs/reference/LoadSyntheaTables.html index f46b5c9..83c07a3 100644 --- a/docs/reference/LoadSyntheaTables.html +++ b/docs/reference/LoadSyntheaTables.html @@ -17,7 +17,7 @@ ETLSyntheaBuilder - 2.0 + 2.1 diff --git a/docs/reference/LoadVocabFromCsv.html b/docs/reference/LoadVocabFromCsv.html index c9f1399..e30a958 100644 --- a/docs/reference/LoadVocabFromCsv.html +++ b/docs/reference/LoadVocabFromCsv.html @@ -17,7 +17,7 @@ ETLSyntheaBuilder - 2.0 + 2.1 @@ -99,7 +99,13 @@

Load Vocabulary Tables From CSV Files.

- 
LoadVocabFromCsv(connectionDetails, cdmSchema, vocabFileLoc, bulkLoad = FALSE)
+ 
LoadVocabFromCsv(
+  connectionDetails,
+  cdmSchema,
+  vocabFileLoc,
+  bulkLoad = FALSE,
+  delimiter = "\t"
+)
@@ -113,7 +119,7 @@

Arguments

cdmSchema

The name of the database schema that will contain the Vocabulary (and CDM) tables. Requires read and write permissions to this database. On SQL -Server, this should specifiy both the database and the schema, +Server, this should specify both the database and the schema, so for example 'cdm_instance.dbo'.

@@ -124,6 +130,10 @@

Arguments

bulkLoad

Boolean flag indicating whether or not to use bulk loading (if possible). Default is FALSE.

+ +
delimiter
+

Parameter to specify the delimiter of the csv file. By default the function expects the file to be tab delimited ("\t") based on the export from Athena.

+

Details

diff --git a/docs/reference/LoadVocabFromSchema.html b/docs/reference/LoadVocabFromSchema.html index 9015b62..341fbd4 100644 --- a/docs/reference/LoadVocabFromSchema.html +++ b/docs/reference/LoadVocabFromSchema.html @@ -17,7 +17,7 @@ ETLSyntheaBuilder - 2.0 + 2.1
diff --git a/docs/reference/TruncateEventTables.html b/docs/reference/TruncateEventTables.html index 2ba94d5..fe945db 100644 --- a/docs/reference/TruncateEventTables.html +++ b/docs/reference/TruncateEventTables.html @@ -17,7 +17,7 @@ ETLSyntheaBuilder - 2.0 + 2.1 diff --git a/docs/reference/TruncateVocabTables.html b/docs/reference/TruncateVocabTables.html index 0d35fd9..5cf99b5 100644 --- a/docs/reference/TruncateVocabTables.html +++ b/docs/reference/TruncateVocabTables.html @@ -17,7 +17,7 @@ ETLSyntheaBuilder - 2.0 + 2.1 diff --git a/docs/reference/backupCDM.html b/docs/reference/backupCDM.html index 8f922a0..d513dc9 100644 --- a/docs/reference/backupCDM.html +++ b/docs/reference/backupCDM.html @@ -17,7 +17,7 @@ ETLSyntheaBuilder - 2.0 + 2.1 diff --git a/docs/reference/createExtraIndices.html b/docs/reference/createExtraIndices.html index ef583d1..dee6bc3 100644 --- a/docs/reference/createExtraIndices.html +++ b/docs/reference/createExtraIndices.html @@ -17,7 +17,7 @@ ETLSyntheaBuilder - 2.0 + 2.1 @@ -91,7 +91,7 @@

Create Optional Extra Indices for ETL Performance

- +
@@ -127,7 +127,7 @@

Arguments

syntheaVersion
-

Your Synthea version. Currently "2.7.0", "3.0.0", "3.1.0" and "3.2.0" are supported.

+

Your Synthea version. Currently "2.7.0", "3.0.0", "3.1.0", "3.2.0" and "3.3.0" are supported.

outputFolder
@@ -142,7 +142,7 @@

Arguments

Details

This function creates indices which have been found to speed up certain long-running INSERT queries in LoadEventTables, for some users. Indices are created on the intermediate vocabulary mapping tables; the person & provider CDM tables; - and the claims_transactions Synthea table (in Synthea 3.0.0, 3.1.0 and 3.2.0) .

+ and the claims_transactions Synthea table (in Synthea 3.0.0, 3.1.0, 3.2.0 and 3.3.0) .

diff --git a/docs/reference/createPrunedTables.html b/docs/reference/createPrunedTables.html index ba64cc7..91258f6 100644 --- a/docs/reference/createPrunedTables.html +++ b/docs/reference/createPrunedTables.html @@ -18,7 +18,7 @@ ETLSyntheaBuilder - 2.0 + 2.1 diff --git a/docs/reference/exportToSQLite.html b/docs/reference/exportToSQLite.html index 884abe7..9899c44 100644 --- a/docs/reference/exportToSQLite.html +++ b/docs/reference/exportToSQLite.html @@ -17,7 +17,7 @@ ETLSyntheaBuilder - 2.0 + 2.1 diff --git a/docs/reference/getEventConceptId.html b/docs/reference/getEventConceptId.html index 1de55da..bc7cd49 100644 --- a/docs/reference/getEventConceptId.html +++ b/docs/reference/getEventConceptId.html @@ -17,7 +17,7 @@ ETLSyntheaBuilder - 2.0 + 2.1 diff --git a/docs/reference/index.html b/docs/reference/index.html index d6e7fc0..6421fbe 100644 --- a/docs/reference/index.html +++ b/docs/reference/index.html @@ -17,7 +17,7 @@ ETLSyntheaBuilder - 2.0 + 2.1 @@ -108,6 +108,10 @@

All functions CreateCDMTables()

Create Common Data Model Tables.

+ +

CreateExtraIndices()

+ +

Create Optional Extra Indices for ETL Performance

CreateMapAndRollupTables()

@@ -168,10 +172,6 @@

All functions backupCDM()

Backup Vocabulary and Event tables before pruning.

- -

CreateExtraIndices()

- -

Create Optional Extra Indices for ETL Performance

createPrunedTables()

diff --git a/docs/reference/pruneCDM.html b/docs/reference/pruneCDM.html index c6d4feb..1d03c92 100644 --- a/docs/reference/pruneCDM.html +++ b/docs/reference/pruneCDM.html @@ -17,7 +17,7 @@ ETLSyntheaBuilder - 2.0 + 2.1 diff --git a/docs/reference/restoreCDMTables.html b/docs/reference/restoreCDMTables.html index 49e5de9..70782b7 100644 --- a/docs/reference/restoreCDMTables.html +++ b/docs/reference/restoreCDMTables.html @@ -17,7 +17,7 @@ ETLSyntheaBuilder - 2.0 + 2.1 diff --git a/docs/sitemap.xml b/docs/sitemap.xml index 1387fdc..44b4d2d 100644 --- a/docs/sitemap.xml +++ b/docs/sitemap.xml @@ -3,30 +3,6 @@ /404.html - - /Condition_occurrence.html - - - /Drug_exposure.html - - - /Measurement.html - - - /Observation.html - - - /Observation_period.html - - - /Person.html - - - /Procedure_occurrence.html - - - /Visit_occurrence.html - /articles/care_site.html @@ -81,9 +57,30 @@ /authors.html + + /condition_occurrence.html + + + /drug_exposure.html + /index.html + + /measurement.html + + + /observation.html + + + /observation_period.html + + + /person.html + + + /procedure_occurrence.html + /readme.html @@ -165,4 +162,7 @@ /reference/restoreCDMTables.html + + /visit_occurrence.html + diff --git a/inst/sql/sql_server/cdm_version/v531/insert_condition_occurrence.sql b/inst/sql/sql_server/cdm_version/v531/insert_condition_occurrence.sql index aaf159c..c34e1e9 100644 --- a/inst/sql/sql_server/cdm_version/v531/insert_condition_occurrence.sql +++ b/inst/sql/sql_server/cdm_version/v531/insert_condition_occurrence.sql @@ -42,12 +42,27 @@ join @cdm_schema.source_to_standard_vocab_map srctostdvm on srctostdvm.source_code = c.code and srctostdvm.target_domain_id = 'Condition' and srctostdvm.target_vocabulary_id = 'SNOMED' + {@synthea_version == "2.7.0" | @synthea_version == "3.0.0" | @synthea_version == "3.1.0" | @synthea_version == "3.2.0"} ? { and srctostdvm.source_vocabulary_id = 'SNOMED' +} + +{@synthea_version == "3.3.0"} ? { + and srctostdvm.source_vocabulary_id in ( 'SNOMED','ICD10CM') +} + and srctostdvm.target_standard_concept = 'S' and srctostdvm.target_invalid_reason is null join @cdm_schema.source_to_source_vocab_map srctosrcvm on srctosrcvm.source_code = c.code + +{@synthea_version == "2.7.0" | @synthea_version == "3.0.0" | @synthea_version == "3.1.0" | @synthea_version == "3.2.0"} ? { and srctosrcvm.source_vocabulary_id = 'SNOMED' +} + +{@synthea_version == "3.3.0"} ? { + and srctosrcvm.source_vocabulary_id in ('SNOMED','ICD10CM') +} + and srctosrcvm.source_domain_id = 'Condition' left join @cdm_schema.final_visit_ids fv on fv.encounter_id = c.encounter diff --git a/inst/sql/sql_server/cdm_version/v531/insert_measurement.sql b/inst/sql/sql_server/cdm_version/v531/insert_measurement.sql index fac11f5..43ae88d 100644 --- a/inst/sql/sql_server/cdm_version/v531/insert_measurement.sql +++ b/inst/sql/sql_server/cdm_version/v531/insert_measurement.sql @@ -53,7 +53,7 @@ select pr.date measurement_time, } -{@synthea_version == "3.0.0" | @synthea_version == "3.1.0" | @synthea_version == "3.2.0" } ? { +{@synthea_version == "3.0.0" | @synthea_version == "3.1.0" | @synthea_version == "3.2.0" | @synthea_version == "3.3.0"} ? { pr.start measurement_date, pr.start measurement_datetime, pr.start measurement_time, @@ -115,7 +115,7 @@ select pr.provider_id provider_id, fv.visit_occurrence_id_new visit_occurrence_id, fv.visit_occurrence_id_new + 1000000 visit_detail_id, - o.code measurement_source_value, + left(o.value,50) measurement_source_value, coalesce(srctosrcvm.source_concept_id,0) measurement_source_concept_id, o.units unit_source_value, o.value value_source_value diff --git a/inst/sql/sql_server/cdm_version/v531/insert_observation.sql b/inst/sql/sql_server/cdm_version/v531/insert_observation.sql index 9c821c8..16a3609 100644 --- a/inst/sql/sql_server/cdm_version/v531/insert_observation.sql +++ b/inst/sql/sql_server/cdm_version/v531/insert_observation.sql @@ -135,7 +135,7 @@ cast(null as varchar) value_as_string, pr.provider_id provider_id, fv.visit_occurrence_id_new visit_occurrence_id, fv.visit_occurrence_id_new + 1000000 visit_detail_id, -o.code observation_source_value, +left(o.value,50) observation_source_value, srctosrcvm.source_concept_id observation_source_concept_id, cast(null as varchar) unit_source_value, cast(null as varchar) qualifier_source_value diff --git a/inst/sql/sql_server/cdm_version/v531/insert_payer_plan_period.sql b/inst/sql/sql_server/cdm_version/v531/insert_payer_plan_period.sql index 286f3d1..94eee96 100644 --- a/inst/sql/sql_server/cdm_version/v531/insert_payer_plan_period.sql +++ b/inst/sql/sql_server/cdm_version/v531/insert_payer_plan_period.sql @@ -33,7 +33,7 @@ from (select CAST(pt.end_year AS DATE) payer_plan_period_end_date, } - {@synthea_version == "3.1.0" | @synthea_version == "3.2.0" } ? { + {@synthea_version == "3.1.0" | @synthea_version == "3.2.0" | @synthea_version == "3.3.0" } ? { CAST(pt.start_date AS DATE) payer_plan_period_start_date, CAST(pt.end_date AS DATE) payer_plan_period_end_date, } diff --git a/inst/sql/sql_server/cdm_version/v531/insert_procedure_occurrence.sql b/inst/sql/sql_server/cdm_version/v531/insert_procedure_occurrence.sql index 0954208..c42ef7e 100644 --- a/inst/sql/sql_server/cdm_version/v531/insert_procedure_occurrence.sql +++ b/inst/sql/sql_server/cdm_version/v531/insert_procedure_occurrence.sql @@ -25,7 +25,7 @@ pr.date procedure_date, pr.date procedure_datetime, } -{@synthea_version == "3.0.0" | @synthea_version == "3.1.0" | @synthea_version == "3.2.0" } ? { +{@synthea_version == "3.0.0" | @synthea_version == "3.1.0" | @synthea_version == "3.2.0" | @synthea_version == "3.3.0" } ? { pr.start procedure_date, pr.start procedure_datetime, } @@ -44,12 +44,23 @@ join @cdm_schema.source_to_standard_vocab_map srctostdvm on srctostdvm.source_code = pr.code and srctostdvm.target_domain_id = 'Procedure' and srctostdvm.target_vocabulary_id = 'SNOMED' - and srctostdvm.source_vocabulary_id = 'SNOMED' +{@synthea_version == "2.7.0" | @synthea_version == "3.0.0" | @synthea_version == "3.1.0" | @synthea_version == "3.2.0" } ? { + and srctostdvm.source_vocabulary_id = 'SNOMED' +} +{@synthea_version == "3.3.0" } ? { + and srctostdvm.source_vocabulary_id in ('SNOMED','HCPCS') +} and srctostdvm.target_standard_concept = 'S' and srctostdvm.target_invalid_reason is null join @cdm_schema.source_to_source_vocab_map srctosrcvm on srctosrcvm.source_code = pr.code +{@synthea_version == "2.7.0" | @synthea_version == "3.0.0" | @synthea_version == "3.1.0" | @synthea_version == "3.2.0" } ? { and srctosrcvm.source_vocabulary_id = 'SNOMED' +} + +{@synthea_version == "3.3.0" } ? { + and srctosrcvm.source_vocabulary_id in ('SNOMED','HCPCS') +} left join @cdm_schema.final_visit_ids fv on fv.encounter_id = pr.encounter left join @synthea_schema.encounters e diff --git a/inst/sql/sql_server/cdm_version/v540/final_visit_ids.sql b/inst/sql/sql_server/cdm_version/v540/final_visit_ids.sql index 8b2a9fa..b5ef5d1 100644 --- a/inst/sql/sql_server/cdm_version/v540/final_visit_ids.sql +++ b/inst/sql/sql_server/cdm_version/v540/final_visit_ids.sql @@ -1,39 +1,30 @@ if object_id('@cdm_schema.FINAL_VISIT_IDS', 'U') is not null drop table @cdm_schema.FINAL_VISIT_IDS; - -SELECT encounter_id, VISIT_OCCURRENCE_ID_NEW -INTO @cdm_schema.FINAL_VISIT_IDS -FROM( - SELECT *, ROW_NUMBER () OVER (PARTITION BY encounter_id ORDER BY PRIORITY) AS RN - FROM ( - SELECT *, - CASE - WHEN encounterclass in ('emergency','urgent') - THEN ( - CASE - WHEN VISIT_TYPE = 'inpatient' AND VISIT_OCCURRENCE_ID_NEW IS NOT NULL - THEN 1 - WHEN VISIT_TYPE in ('emergency','urgent') AND VISIT_OCCURRENCE_ID_NEW IS NOT NULL - THEN 2 - ELSE 99 - END - ) - WHEN encounterclass in ('ambulatory', 'wellness', 'outpatient') - THEN ( - CASE - WHEN VISIT_TYPE = 'inpatient' AND VISIT_OCCURRENCE_ID_NEW IS NOT NULL - THEN 1 - WHEN VISIT_TYPE in ('ambulatory', 'wellness', 'outpatient') AND VISIT_OCCURRENCE_ID_NEW IS NOT NULL - THEN 2 - ELSE 99 - END - ) - WHEN encounterclass = 'inpatient' AND VISIT_TYPE = 'inpatient' AND VISIT_OCCURRENCE_ID_NEW IS NOT NULL - THEN 1 - ELSE 99 - END AS PRIORITY - FROM @cdm_schema.ASSIGN_ALL_VISIT_IDS - ) T1 -) T2 -WHERE RN=1 +CREATE TABLE @cdm_schema.FINAL_VISIT_IDS AS + SELECT encounter_id, VISIT_OCCURRENCE_ID_NEW + FROM ( + SELECT *, + ROW_NUMBER() OVER (PARTITION BY encounter_id ORDER BY PRIORITY) AS RN + FROM ( + SELECT *, + CASE + WHEN encounterclass IN ('emergency', 'urgent') THEN + CASE + WHEN VISIT_TYPE = 'inpatient' AND VISIT_OCCURRENCE_ID_NEW IS NOT NULL THEN 1 + WHEN VISIT_TYPE IN ('emergency', 'urgent') AND VISIT_OCCURRENCE_ID_NEW IS NOT NULL THEN 2 + ELSE 99 + END + WHEN encounterclass IN ('ambulatory', 'wellness', 'outpatient') THEN + CASE + WHEN VISIT_TYPE = 'inpatient' AND VISIT_OCCURRENCE_ID_NEW IS NOT NULL THEN 1 + WHEN VISIT_TYPE IN ('ambulatory', 'wellness', 'outpatient') AND VISIT_OCCURRENCE_ID_NEW IS NOT NULL THEN 2 + ELSE 99 + END + WHEN encounterclass = 'inpatient' AND VISIT_TYPE = 'inpatient' AND VISIT_OCCURRENCE_ID_NEW IS NOT NULL THEN 1 + ELSE 99 + END AS PRIORITY + FROM @cdm_schema.ASSIGN_ALL_VISIT_IDS + ) T1 + ) RankedVisits + WHERE RN = 1; diff --git a/inst/sql/sql_server/cdm_version/v540/insert_condition_occurrence.sql b/inst/sql/sql_server/cdm_version/v540/insert_condition_occurrence.sql index 434a405..53e0d4f 100644 --- a/inst/sql/sql_server/cdm_version/v540/insert_condition_occurrence.sql +++ b/inst/sql/sql_server/cdm_version/v540/insert_condition_occurrence.sql @@ -36,25 +36,40 @@ c.code condition_source_value, srctosrcvm.source_concept_id condition_source_concept_id, null condition_status_source_value, 0 condition_status_concept_id - from @synthea_schema.conditions c join @cdm_schema.source_to_standard_vocab_map srctostdvm on srctostdvm.source_code = c.code and srctostdvm.target_domain_id = 'Condition' and srctostdvm.target_vocabulary_id = 'SNOMED' + +{@synthea_version == "2.7.0" | @synthea_version == "3.0.0" | @synthea_version == "3.1.0" | @synthea_version == "3.2.0"} ? { and srctostdvm.source_vocabulary_id = 'SNOMED' +} + +{@synthea_version == "3.3.0"} ? { + and srctostdvm.source_vocabulary_id in ( 'SNOMED','ICD10CM') +} + and srctostdvm.target_standard_concept = 'S' and srctostdvm.target_invalid_reason is null join @cdm_schema.source_to_source_vocab_map srctosrcvm on srctosrcvm.source_code = c.code + +{@synthea_version == "2.7.0" | @synthea_version == "3.0.0" | @synthea_version == "3.1.0" | @synthea_version == "3.2.0"} ? { and srctosrcvm.source_vocabulary_id = 'SNOMED' +} + +{@synthea_version == "3.3.0"} ? { + and srctosrcvm.source_vocabulary_id in ('SNOMED','ICD10CM') +} + and srctosrcvm.source_domain_id = 'Condition' left join @cdm_schema.final_visit_ids fv on fv.encounter_id = c.encounter left join @synthea_schema.encounters e on c.encounter = e.id and c.patient = e.patient -left join @cdm_schema.provider pr +left join @cdm_schema.provider pr on e.provider = pr.provider_source_value join @cdm_schema.person p on c.patient = p.person_source_value diff --git a/inst/sql/sql_server/cdm_version/v540/insert_drug_exposure.sql b/inst/sql/sql_server/cdm_version/v540/insert_drug_exposure.sql index ee97d2e..4777e3e 100644 --- a/inst/sql/sql_server/cdm_version/v540/insert_drug_exposure.sql +++ b/inst/sql/sql_server/cdm_version/v540/insert_drug_exposure.sql @@ -77,20 +77,20 @@ cast(null as varchar) route_source_value, cast(null as varchar) dose_unit_source_value from @synthea_schema.medications m join @cdm_schema.source_to_standard_vocab_map srctostdvm - on srctostdvm.source_code = m.code + on srctostdvm.source_code = CAST(m.code as VARCHAR) and srctostdvm.target_domain_id = 'Drug' and srctostdvm.target_vocabulary_id = 'RxNorm' and srctostdvm.target_standard_concept = 'S' and srctostdvm.target_invalid_reason is null join @cdm_schema.source_to_source_vocab_map srctosrcvm - on srctosrcvm.source_code = m.code + on srctosrcvm.source_code = CAST(m.code as VARCHAR) and srctosrcvm.source_vocabulary_id = 'RxNorm' left join @cdm_schema.final_visit_ids fv on fv.encounter_id = m.encounter left join @synthea_schema.encounters e on m.encounter = e.id and m.patient = e.patient -left join @cdm_schema.provider pr +left join @cdm_schema.provider pr on e.provider = pr.provider_source_value join @cdm_schema.person p on p.person_source_value = m.patient @@ -112,30 +112,30 @@ cast(null as varchar) stop_reason, 0 days_supply, cast(null as varchar) sig, 0 route_concept_id, -0 lot_number, +0 lot_number, pr.provider_id provider_id, fv.visit_occurrence_id_new visit_occurrence_id, fv.visit_occurrence_id_new + 1000000 visit_detail_id, i.code drug_source_value, -srctosrcvm.source_concept_id drug_source_concept_id, +srctosrcvm.source_concept_id drug_source_concept_id, cast(null as varchar) route_source_value, cast(null as varchar) dose_unit_source_value from @synthea_schema.immunizations i join @cdm_schema.source_to_standard_vocab_map srctostdvm - on srctostdvm.source_code = i.code + on srctostdvm.source_code = CAST(i.code as VARCHAR) and srctostdvm.target_domain_id = 'Drug' and srctostdvm.target_vocabulary_id = 'CVX' and srctostdvm.target_standard_concept = 'S' and srctostdvm.target_invalid_reason is null join @cdm_schema.source_to_source_vocab_map srctosrcvm - on srctosrcvm.source_code = i.code + on srctosrcvm.source_code = CAST(i.code as VARCHAR) and srctosrcvm.source_vocabulary_id = 'CVX' left join @cdm_schema.final_visit_ids fv on fv.encounter_id = i.encounter left join @synthea_schema.encounters e on i.encounter = e.id and i.patient = e.patient -left join @cdm_schema.provider pr +left join @cdm_schema.provider pr on e.provider = pr.provider_source_value join @cdm_schema.person p on p.person_source_value = i.patient diff --git a/inst/sql/sql_server/cdm_version/v540/insert_measurement.sql b/inst/sql/sql_server/cdm_version/v540/insert_measurement.sql index 32893f0..b630933 100644 --- a/inst/sql/sql_server/cdm_version/v540/insert_measurement.sql +++ b/inst/sql/sql_server/cdm_version/v540/insert_measurement.sql @@ -59,7 +59,7 @@ select pr.date measurement_time, } -{@synthea_version == "3.0.0" | @synthea_version == "3.1.0" | @synthea_version == "3.2.0" } ? { +{@synthea_version == "3.0.0" | @synthea_version == "3.1.0" | @synthea_version == "3.2.0" | @synthea_version == "3.3.0"} ? { pr.start measurement_date, pr.start measurement_datetime, pr.start measurement_time, @@ -128,7 +128,7 @@ select o.code measurement_source_value, coalesce(srctosrcvm.source_concept_id,0) measurement_source_concept_id, o.units unit_source_value, - o.value value_source_value, + left(o.value,50) value_source_value, cast(null as int) unit_source_concept_id, cast(null as bigint) measurement_event_id, cast(null as int) meas_event_field_concept_id diff --git a/inst/sql/sql_server/cdm_version/v540/insert_observation.sql b/inst/sql/sql_server/cdm_version/v540/insert_observation.sql index 495ffdf..d45f059 100644 --- a/inst/sql/sql_server/cdm_version/v540/insert_observation.sql +++ b/inst/sql/sql_server/cdm_version/v540/insert_observation.sql @@ -69,13 +69,13 @@ cast(null as bigint) observation_event_id, cast(null as int) obs_event_field_concept_id from @synthea_schema.allergies a join @cdm_schema.source_to_standard_vocab_map srctostdvm - on srctostdvm.source_code = a.code + on srctostdvm.source_code = CAST(a.code as VARCHAR) and srctostdvm.target_domain_id = 'Observation' and srctostdvm.target_vocabulary_id = 'SNOMED' and srctostdvm.target_standard_concept = 'S' and srctostdvm.target_invalid_reason is null join @cdm_schema.source_to_source_vocab_map srctosrcvm - on srctosrcvm.source_code = a.code + on srctosrcvm.source_code = CAST(a.code as VARCHAR) and srctosrcvm.source_vocabulary_id = 'SNOMED' and srctosrcvm.source_domain_id = 'Observation' left join @cdm_schema.final_visit_ids fv @@ -83,7 +83,7 @@ left join @cdm_schema.final_visit_ids fv left join @synthea_schema.encounters e on a.encounter = e.id and a.patient = e.patient -left join @cdm_schema.provider pr +left join @cdm_schema.provider pr on e.provider = pr.provider_source_value join @cdm_schema.person p on p.person_source_value = a.patient @@ -114,13 +114,13 @@ cast(null as int) obs_event_field_concept_id from @synthea_schema.conditions c join @cdm_schema.source_to_standard_vocab_map srctostdvm - on srctostdvm.source_code = c.code + on srctostdvm.source_code = CAST(c.code as VARCHAR) and srctostdvm.target_domain_id = 'Observation' and srctostdvm.target_vocabulary_id = 'SNOMED' and srctostdvm.target_standard_concept = 'S' and srctostdvm.target_invalid_reason is null join @cdm_schema.source_to_source_vocab_map srctosrcvm - on srctosrcvm.source_code = c.code + on srctosrcvm.source_code = CAST(c.code as VARCHAR) and srctosrcvm.source_vocabulary_id = 'SNOMED' and srctosrcvm.source_domain_id = 'Observation' left join @cdm_schema.final_visit_ids fv @@ -128,11 +128,11 @@ left join @cdm_schema.final_visit_ids fv left join @synthea_schema.encounters e on c.encounter = e.id and c.patient = e.patient -left join @cdm_schema.provider pr +left join @cdm_schema.provider pr on e.provider = pr.provider_source_value join @cdm_schema.person p on p.person_source_value = c.patient - + union all select @@ -153,19 +153,19 @@ o.code observation_source_value, srctosrcvm.source_concept_id observation_source_concept_id, cast(null as varchar) unit_source_value, cast(null as varchar) qualifier_source_value, -cast(null as varchar) value_source_value, +left(o.value,50) value_source_value, cast(null as bigint) observation_event_id, cast(null as int) obs_event_field_concept_id from @synthea_schema.observations o join @cdm_schema.source_to_standard_vocab_map srctostdvm - on srctostdvm.source_code = o.code + on srctostdvm.source_code = CAST(o.code as VARCHAR) and srctostdvm.target_domain_id = 'Observation' and srctostdvm.target_vocabulary_id = 'LOINC' and srctostdvm.target_standard_concept = 'S' and srctostdvm.target_invalid_reason is null join @cdm_schema.source_to_source_vocab_map srctosrcvm - on srctosrcvm.source_code = o.code + on srctosrcvm.source_code = CAST(o.code as VARCHAR) and srctosrcvm.source_vocabulary_id = 'LOINC' and srctosrcvm.source_domain_id = 'Observation' left join @cdm_schema.final_visit_ids fv @@ -173,10 +173,9 @@ left join @cdm_schema.final_visit_ids fv left join @synthea_schema.encounters e on o.encounter = e.id and o.patient = e.patient -left join @cdm_schema.provider pr +left join @cdm_schema.provider pr on e.provider = pr.provider_source_value join @cdm_schema.person p on p.person_source_value = o.patient - ) tmp ; diff --git a/inst/sql/sql_server/cdm_version/v540/insert_payer_plan_period.sql b/inst/sql/sql_server/cdm_version/v540/insert_payer_plan_period.sql index e5ccaa8..ff33df3 100644 --- a/inst/sql/sql_server/cdm_version/v540/insert_payer_plan_period.sql +++ b/inst/sql/sql_server/cdm_version/v540/insert_payer_plan_period.sql @@ -23,10 +23,9 @@ select ROW_NUMBER()OVER(ORDER BY person_id, payer_plan_period_start_date) payer_ * from (select per.person_id person_id, - {@synthea_version == "2.7.0"} ? { - CAST(CONCAT('01-JAN-',CAST(pt.start_year AS VARCHAR)) AS DATE) payer_plan_period_start_date, - CAST(CONCAT('31-DEC-',CAST(pt.end_year AS VARCHAR)) AS DATE) payer_plan_period_end_date, + CAST(CONCAT(CAST(pt.start_year AS VARCHAR),'-01-01') AS DATE) payer_plan_period_start_date, + CAST(CONCAT(CAST(pt.end_year AS VARCHAR),'-12-31') AS DATE) payer_plan_period_end_date, } {@synthea_version == "3.0.0"} ? { @@ -34,7 +33,7 @@ from (select CAST(pt.end_year AS DATE) payer_plan_period_end_date, } - {@synthea_version == "3.1.0" | @synthea_version == "3.2.0" } ? { + {@synthea_version == "3.1.0" | @synthea_version == "3.2.0" | @synthea_version == "3.3.0" } ? { CAST(pt.start_date AS DATE) payer_plan_period_start_date, CAST(pt.end_date AS DATE) payer_plan_period_end_date, } diff --git a/inst/sql/sql_server/cdm_version/v540/insert_procedure_occurrence.sql b/inst/sql/sql_server/cdm_version/v540/insert_procedure_occurrence.sql index 224a8ab..ad21557 100644 --- a/inst/sql/sql_server/cdm_version/v540/insert_procedure_occurrence.sql +++ b/inst/sql/sql_server/cdm_version/v540/insert_procedure_occurrence.sql @@ -29,7 +29,7 @@ pr.date procedure_end_date, pr.date procedure_end_datetime, } -{@synthea_version == "3.0.0" | @synthea_version == "3.1.0" | @synthea_version == "3.2.0" } ? { +{@synthea_version == "3.0.0" | @synthea_version == "3.1.0" | @synthea_version == "3.2.0" | @synthea_version == "3.3.0" } ? { pr.start procedure_date, pr.start procedure_datetime, pr.stop procedure_end_date, @@ -50,12 +50,25 @@ join @cdm_schema.source_to_standard_vocab_map srctostdvm on srctostdvm.source_code = pr.code and srctostdvm.target_domain_id = 'Procedure' and srctostdvm.target_vocabulary_id = 'SNOMED' - and srctostdvm.source_vocabulary_id = 'SNOMED' + +{@synthea_version == "2.7.0" | @synthea_version == "3.0.0" | @synthea_version == "3.1.0" | @synthea_version == "3.2.0" } ? { + and srctostdvm.source_vocabulary_id = 'SNOMED' +} +{@synthea_version == "3.3.0" } ? { + and srctostdvm.source_vocabulary_id in ('SNOMED','HCPCS') +} and srctostdvm.target_standard_concept = 'S' and srctostdvm.target_invalid_reason is null join @cdm_schema.source_to_source_vocab_map srctosrcvm on srctosrcvm.source_code = pr.code +{@synthea_version == "2.7.0" | @synthea_version == "3.0.0" | @synthea_version == "3.1.0" | @synthea_version == "3.2.0" } ? { and srctosrcvm.source_vocabulary_id = 'SNOMED' +} + +{@synthea_version == "3.3.0" } ? { + and srctosrcvm.source_vocabulary_id in ('SNOMED','HCPCS') +} + left join @cdm_schema.final_visit_ids fv on fv.encounter_id = pr.encounter left join @synthea_schema.encounters e diff --git a/inst/sql/sql_server/extra_indices.sql b/inst/sql/sql_server/extra_indices.sql index 3bbdb97..f44ab8c 100644 --- a/inst/sql/sql_server/extra_indices.sql +++ b/inst/sql/sql_server/extra_indices.sql @@ -31,7 +31,7 @@ CREATE INDEX person_psv ON @cdmDatabaseSchema.person ( person_source_value ); -{@syntheaVersion == '3.0.0' | @syntheaVersion == '3.1.0' | @syntheaVersion == '3.2.0' }?{ +{@syntheaVersion == '3.0.0' | @syntheaVersion == '3.1.0' | @syntheaVersion == '3.2.0' | @syntheaVersion == '3.3.0' }?{ CREATE INDEX claims_transactions_cpap ON @syntheaSchema.claims_transactions ( claimid, patientid, diff --git a/inst/sql/sql_server/synthea_version/v330/create_synthea_tables.sql b/inst/sql/sql_server/synthea_version/v330/create_synthea_tables.sql new file mode 100644 index 0000000..1eedc46 --- /dev/null +++ b/inst/sql/sql_server/synthea_version/v330/create_synthea_tables.sql @@ -0,0 +1,329 @@ + +--HINT DISTRIBUTE_ON_RANDOM +create table @synthea_schema.allergies ( +start date, +stop date, +patient varchar(1000), +encounter varchar(1000), +code varchar(100), +system varchar(255), +description varchar(255), +"type" varchar(255), +category varchar(255), +reaction1 varchar(255), +description1 varchar(255), +severity1 varchar(255), +reaction2 varchar(255), +description2 varchar(255), +severity2 varchar(255) +); + +--HINT DISTRIBUTE_ON_RANDOM +create table @synthea_schema.careplans ( +id varchar(1000), +start date, +stop date, +patient varchar(1000), +encounter varchar(1000), +code varchar(100), +description varchar(255), +reasoncode varchar(255), +reasondescription varchar(255) +); + +--HINT DISTRIBUTE_ON_RANDOM +create table @synthea_schema.conditions ( +start date, +stop date, +patient varchar(1000), +encounter varchar(1000), +system varchar(1000), +code varchar(100), +description varchar(255) +); + +--HINT DISTRIBUTE_ON_RANDOM +create table @synthea_schema.encounters ( +id varchar(1000), +start date, +stop date, +patient varchar(1000), +organization varchar(1000), +provider varchar(1000), +payer varchar(1000), +encounterclass varchar(1000), +code varchar(100), +description varchar(255), +base_encounter_cost numeric, +total_claim_cost numeric, +payer_coverage numeric, +reasoncode varchar(100), +reasondescription varchar(255) +); + +--HINT DISTRIBUTE_ON_RANDOM +create table @synthea_schema.immunizations ( +"date" date, +patient varchar(1000), +encounter varchar(1000), +code varchar(100), +description varchar(255), +base_cost numeric +); + +--HINT DISTRIBUTE_ON_RANDOM +create table @synthea_schema.imaging_studies ( +id varchar(1000), +"date" date, +patient varchar(1000), +encounter varchar(1000), +series_uid varchar(1000), +bodysite_code varchar(100), +bodysite_description varchar(255), +modality_code varchar(100), +modality_description varchar(255), +instance_uid varchar(1000), +SOP_code varchar(100), +SOP_description varchar(255), +procedure_code varchar(255) +); + +--HINT DISTRIBUTE_ON_RANDOM +create table @synthea_schema.medications ( +start date, +stop date, +patient varchar(1000), +payer varchar(1000), +encounter varchar(1000), +code varchar(100), +description varchar(1000), +base_cost numeric, +payer_coverage numeric, +dispenses int, +totalcost numeric, +reasoncode varchar(100), +reasondescription varchar(255) +); + +--HINT DISTRIBUTE_ON_RANDOM +create table @synthea_schema.observations ( +"date" date, +patient varchar(1000), +encounter varchar(1000), +category varchar(1000), +code varchar(100), +description varchar(255), +value varchar(1000), +units varchar(100), +"type" varchar(100) +); + +--HINT DISTRIBUTE_ON_RANDOM +create table @synthea_schema.organizations ( +id varchar(1000), +"name" varchar(1000), +address varchar(1000), +city varchar(100), +state varchar(100), +zip varchar(100), +lat numeric, +lon numeric, +phone varchar(100), +revenue numeric, +utilization varchar(100) +); + +--HINT DISTRIBUTE_ON_RANDOM +create table @synthea_schema.patients ( +id varchar(1000), +birthdate date, +deathdate date, +ssn varchar(100), +drivers varchar(100), +passport varchar(100), +prefix varchar(100), +first varchar(100), +middle varchar(100), +last varchar(100), +suffix varchar(100), +maiden varchar(100), +marital varchar(100), +race varchar(100), +ethnicity varchar(100), +gender varchar(100), +birthplace varchar(100), +address varchar(100), +city varchar(100), +state varchar(100), +county varchar(100), +fips varchar(100), +zip varchar(100), +lat numeric, +lon numeric, +healthcare_expenses numeric, +healthcare_coverage numeric, +income int +); + +--HINT DISTRIBUTE_ON_RANDOM +create table @synthea_schema.procedures ( +start date, +stop date, +patient varchar(1000), +encounter varchar(1000), +system varchar(100), +code varchar(100), +description varchar(255), +base_cost numeric, +reasoncode varchar(1000), +reasondescription varchar(1000) +); + +--HINT DISTRIBUTE_ON_RANDOM +create table @synthea_schema.providers ( +id varchar(1000), +organization varchar(1000), +"name" varchar(100), +gender varchar(100), +speciality varchar(100), +address varchar(255), +city varchar(100), +state varchar(100), +zip varchar(100), +lat numeric, +lon numeric, +encounters int, +"procedures" int +); + +--HINT DISTRIBUTE_ON_RANDOM +create table @synthea_schema.devices ( +start date, +stop date, +patient varchar(1000), +encounter varchar(1000), +code varchar(100), +description varchar(255), +udi varchar(255) +); + +--HINT DISTRIBUTE_ON_RANDOM +create table @synthea_schema.claims ( + id varchar(1000), + patientid varchar(1000), + providerid varchar(1000), + primarypatientinsuranceid varchar(1000), + secondarypatientinsuranceid varchar(1000), + departmentid varchar(1000), + patientdepartmentid varchar(1000), + diagnosis1 varchar(1000), + diagnosis2 varchar(1000), + diagnosis3 varchar(1000), + diagnosis4 varchar(1000), + diagnosis5 varchar(1000), + diagnosis6 varchar(1000), + diagnosis7 varchar(1000), + diagnosis8 varchar(1000), + referringproviderid varchar(1000), + appointmentid varchar(1000), + currentillnessdate date, + servicedate date, + supervisingproviderid varchar(1000), + status1 varchar(1000), + status2 varchar(1000), + statusp varchar(1000), + outstanding1 numeric, + outstanding2 numeric, + outstandingp numeric, + lastbilleddate1 date, + lastbilleddate2 date, + lastbilleddatep date, + healthcareclaimtypeid1 numeric, + healthcareclaimtypeid2 numeric +); + +--HINT DISTRIBUTE_ON_RANDOM +create table @synthea_schema.claims_transactions ( + id varchar(1000), + claimid varchar(1000), + chargeid numeric, + patientid varchar(1000), + "type" varchar(1000), + amount numeric, + method varchar(1000), + fromdate date, + todate date, + placeofservice varchar(1000), + procedurecode varchar(1000), + modifier1 varchar(1000), + modifier2 varchar(1000), + diagnosisref1 numeric, + diagnosisref2 numeric, + diagnosisref3 numeric, + diagnosisref4 numeric, + units numeric, + departmentid numeric, + notes varchar(1000), + unitamount numeric, + transferoutid numeric, + transfertype varchar(1000), + payments numeric, + adjustments numeric, + transfers numeric, + outstanding numeric, + appointmentid varchar(1000), + linenote varchar(1000), + patientinsuranceid varchar(1000), + feescheduleid numeric, + providerid varchar(1000), + supervisingproviderid varchar(1000) +); + +--HINT DISTRIBUTE_ON_RANDOM +create table @synthea_schema.payer_transitions ( + patient varchar(1000), + memberid varchar(1000), + start_date date, + end_date date, + payer varchar(1000), + secondary_payer varchar(1000), + plan_ownership varchar(1000), + owner_name varchar(1000) +); + +--HINT DISTRIBUTE_ON_RANDOM +create table @synthea_schema.payers ( + id varchar(1000), + "name" varchar(1000), + ownership varchar NULL, + address varchar(1000), + city varchar(1000), + state_headquartered varchar(1000), + zip varchar(1000), + phone varchar(1000), + amount_covered numeric, + amount_uncovered numeric, + revenue numeric, + covered_encounters numeric, + uncovered_encounters numeric, + covered_medications numeric, + uncovered_medications numeric, + covered_procedures numeric, + uncovered_procedures numeric, + covered_immunizations numeric, + uncovered_immunizations numeric, + unique_customers numeric, + qols_avg numeric, + member_months numeric +); + +--HINT DISTRIBUTE_ON_RANDOM +create table @synthea_schema.supplies ( + "date" date, + patient varchar(1000), + encounter varchar(1000), + code varchar(1000), + description varchar(1000), + quantity numeric +); + diff --git a/man/CreateMapAndRollupTables.Rd b/man/CreateMapAndRollupTables.Rd index b4732da..78c5560 100644 --- a/man/CreateMapAndRollupTables.Rd +++ b/man/CreateMapAndRollupTables.Rd @@ -36,7 +36,7 @@ so for example 'cdm_instance.dbo'.} \item{cdmVersion}{The version of your CDM. Currently "5.3" and "5.4".} \item{syntheaVersion}{The version of Synthea used to generate the csv files. -Currently "2.7.0", "3.0.0", "3.1.0" and "3.2.0" are supported.} +Currently "2.7.0", "3.0.0", "3.1.0", "3.2.0" and "3.3.0" are supported.} \item{cdmSourceName}{The source name to insert into the CDM_SOURCE table. Default is Synthea synthetic health database.} diff --git a/man/CreateSyntheaTables.Rd b/man/CreateSyntheaTables.Rd index 03a4d72..eb0b5a2 100644 --- a/man/CreateSyntheaTables.Rd +++ b/man/CreateSyntheaTables.Rd @@ -17,7 +17,7 @@ Server, this should specify both the database and the schema, so for example 'cdm_instance.dbo'.} \item{syntheaVersion}{The version of Synthea used to generate the csv files. -Currently "2.7.0", "3.0.0", "3.1.0", and "3.2.0" are is supported.} +Currently "2.7.0", "3.0.0", "3.1.0", "3.2.0" and "3.3.0" are is supported.} } \description{ This function creates all Synthea tables. diff --git a/man/CreateVocabMapTables.Rd b/man/CreateVocabMapTables.Rd index ef0967b..aade9b8 100644 --- a/man/CreateVocabMapTables.Rd +++ b/man/CreateVocabMapTables.Rd @@ -13,7 +13,7 @@ function \code{createConnectionDetails} in the \item{cdmSchema}{The name of the database schema that will contain the Vocab mapping tables. Requires read and write permissions to this database. On SQL -Server, this should specifiy both the database and the schema, +Server, this should specify both the database and the schema, so for example 'cdm_instance.dbo'.} \item{cdmVersion}{The version of your CDM. Currently "5.3" and "5.4" are supported.} diff --git a/man/LoadEventTables.Rd b/man/LoadEventTables.Rd index 64dcbf9..e019808 100644 --- a/man/LoadEventTables.Rd +++ b/man/LoadEventTables.Rd @@ -37,7 +37,7 @@ so for example 'cdm_instance.dbo'.} \item{cdmVersion}{The version of your CDM. Currently "5.3" and "5.4".} \item{syntheaVersion}{The version of Synthea used to generate the csv files. -Currently "2.7.0","3.0.0","3.1.0" and "3.2.0" are supported.} +Currently "2.7.0","3.0.0","3.1.0","3.2.0" and "3.3.0" are supported.} \item{cdmSourceName}{The source name to insert into the CDM_SOURCE table. Default is Synthea synthetic health database.} diff --git a/man/LoadVocabFromCsv.Rd b/man/LoadVocabFromCsv.Rd index 3dccf3e..b0ac282 100644 --- a/man/LoadVocabFromCsv.Rd +++ b/man/LoadVocabFromCsv.Rd @@ -4,7 +4,13 @@ \alias{LoadVocabFromCsv} \title{Load Vocabulary Tables From CSV Files.} \usage{ -LoadVocabFromCsv(connectionDetails, cdmSchema, vocabFileLoc, bulkLoad = FALSE) +LoadVocabFromCsv( + connectionDetails, + cdmSchema, + vocabFileLoc, + bulkLoad = FALSE, + delimiter = "\\t" +) } \arguments{ \item{connectionDetails}{An R object of type\cr\code{connectionDetails} created using the @@ -13,12 +19,14 @@ function \code{createConnectionDetails} in the \item{cdmSchema}{The name of the database schema that will contain the Vocabulary (and CDM) tables. Requires read and write permissions to this database. On SQL -Server, this should specifiy both the database and the schema, +Server, this should specify both the database and the schema, so for example 'cdm_instance.dbo'.} \item{vocabFileLoc}{The location of the vocabulary csv files.} \item{bulkLoad}{Boolean flag indicating whether or not to use bulk loading (if possible). Default is FALSE.} + +\item{delimiter}{Parameter to specify the delimiter of the csv file. By default the function expects the file to be tab delimited ("\\t") based on the export from Athena.} } \description{ This function populates all Vocabulary tables with data in csv files. diff --git a/man/createExtraIndices.Rd b/man/createExtraIndices.Rd index 03cfdc8..951e686 100644 --- a/man/createExtraIndices.Rd +++ b/man/createExtraIndices.Rd @@ -23,7 +23,7 @@ Server, this should specify both the database and the schema, so for example 'cd \item{syntheaSchema}{The name of the Synthea database schema. Requires read and write permissions to this schema. On SQL Server, this should specify both the database and the schema, so for example 'synthea.dbo'.} -\item{syntheaVersion}{Your Synthea version. Currently "2.7.0", "3.0.0", "3.1.0" and "3.2.0" are supported.} +\item{syntheaVersion}{Your Synthea version. Currently "2.7.0", "3.0.0", "3.1.0", "3.2.0" and "3.3.0" are supported.} \item{outputFolder}{Location of the SQL scripts if sqlOnly = TRUE. Default is NULL.} @@ -35,5 +35,5 @@ This function creates indices for certain tables which may help to speed up Load \details{ This function creates indices which have been found to speed up certain long-running INSERT queries in LoadEventTables, for some users. Indices are created on the intermediate vocabulary mapping tables; the person & provider CDM tables; - and the claims_transactions Synthea table (in Synthea 3.0.0, 3.1.0 and 3.2.0) . + and the claims_transactions Synthea table (in Synthea 3.0.0, 3.1.0, 3.2.0 and 3.3.0) . }