Add support for synthea v3.3 and various fixes (#202)

* fixes for duckdb support * synthea v3.3.0 support (addition of icd10 codes to condition_occurrence logic) * update of LoadFromVocabCSV.R function to accept a delimiter #198 Co-authored-by: Frank DeFalco <[email protected]>
OHDSI · Oct 4, 2024 · e59d1c7 · e59d1c7
1 parent 48c33f8
commit e59d1c7
Show file tree

Hide file tree

Showing 76 changed files with 583 additions and 187 deletions.
diff --git a/.gitignore b/.gitignore
@@ -1,4 +1,5 @@
 output/*.sql
+output/*.txt
 .DS_Store
 .Rproj.user
 errorReport.txt

diff --git a/DESCRIPTION b/DESCRIPTION
@@ -1,7 +1,7 @@
 Package: ETLSyntheaBuilder
 Type: Package
 Title: A Builder for Converting the Synthea Data to the OMOP CDM
-Version: 2.0
+Version: 2.1
 mDate: 2021-12-04
 Author: Anthony Molinaro [aut, cre],
         Clair Blacketer [aut],
@@ -28,7 +28,7 @@ Remotes:
   OHDSI/CommonDataModel
 Encoding: UTF-8
 LazyData: true
-RoxygenNote: 7.2.3
+RoxygenNote: 7.3.1
 Suggests: 
     rmarkdown,
     knitr,

diff --git a/R/CreateMapAndRollupTables.r b/R/CreateMapAndRollupTables.r
@@ -19,7 +19,7 @@
 #'                                     so for example 'cdm_instance.dbo'.
 #' @param cdmVersion The version of your CDM.  Currently "5.3" and "5.4".
 #' @param syntheaVersion The version of Synthea used to generate the csv files.
-#'                       Currently "2.7.0", "3.0.0", "3.1.0" and "3.2.0" are supported.
+#'                       Currently "2.7.0", "3.0.0", "3.1.0", "3.2.0" and "3.3.0" are supported.
 #' @param cdmSourceName	The source name to insert into the CDM_SOURCE table.  Default is Synthea synthetic health database.
 #' @param cdmSourceAbbreviation The source abbreviation to insert into the CDM_SOURCE table.  Default is Synthea.
 #' @param cdmHolder The holder to insert into the CDM_SOURCE table.  Default is OHDSI
@@ -46,11 +46,11 @@ CreateMapAndRollupTables <- function(connectionDetails,
     stop("Unsupported CDM specified. Supported CDM versions are \"5.3\" and \"5.4\".")
   }
 
-  supportedSyntheaVersions <- c("2.7.0", "3.0.0", "3.1.0", "3.2.0")
+  supportedSyntheaVersions <- c("2.7.0", "3.0.0", "3.1.0", "3.2.0", "3.3.0")
 
   if (!(syntheaVersion %in% supportedSyntheaVersions))
     stop(
-      "Invalid Synthea version specified. Currently \"2.7.0\", \"3.0.0\",\"3.1.0\", and \"3.2.0\" are supported."
+      "Invalid Synthea version specified. Currently \"2.7.0\", \"3.0.0\",\"3.1.0\", \"3.2.0\", and \"3.3.0\" are supported."
     )
 
   # Create Vocabulary mapping tables

diff --git a/R/CreateSyntheaTables.r b/R/CreateSyntheaTables.r
@@ -10,7 +10,7 @@
 #'                                     Server, this should specify both the database and the schema,
 #'                                     so for example 'cdm_instance.dbo'.
 #' @param syntheaVersion The version of Synthea used to generate the csv files.
-#'                       Currently "2.7.0", "3.0.0", "3.1.0", and "3.2.0" are is supported.
+#'                       Currently "2.7.0", "3.0.0", "3.1.0", "3.2.0" and "3.3.0" are is supported.
 #'
 #'@export
 
@@ -29,9 +29,11 @@ CreateSyntheaTables <-
       sqlFilePath <- "synthea_version/v310"
     else if (syntheaVersion == "3.2.0")
       sqlFilePath <- "synthea_version/v320"
+    else if (syntheaVersion == "3.3.0")
+      sqlFilePath <- "synthea_version/v330"
     else
       stop(
-        "Invalid synthea version specified. Currently \"2.7.0\", \"3.0.0\", \"3.1.0\" and \"3.2.0\" are supported."
+        "Invalid synthea version specified. Currently \"2.7.0\", \"3.0.0\", \"3.1.0\" , \"3.2.0\" and \"3.3.0\" are supported."
       )
 
     sqlFilename <-

diff --git a/R/LoadEventTables.r b/R/LoadEventTables.r
@@ -18,7 +18,7 @@
 #'                                     so for example 'cdm_instance.dbo'.
 #' @param cdmVersion The version of your CDM.  Currently "5.3" and "5.4".
 #' @param syntheaVersion The version of Synthea used to generate the csv files.
-#'                       Currently "2.7.0","3.0.0","3.1.0" and "3.2.0" are supported.
+#'                       Currently "2.7.0","3.0.0","3.1.0","3.2.0" and "3.3.0" are supported.
 #' @param cdmSourceName	The source name to insert into the CDM_SOURCE table.  Default is Synthea synthetic health database.
 #' @param cdmSourceAbbreviation The source abbreviation to insert into the CDM_SOURCE table.  Default is Synthea.
 #' @param cdmHolder The holder to insert into the CDM_SOURCE table.  Default is OHDSI
@@ -51,11 +51,11 @@ LoadEventTables <- function(connectionDetails,
     stop("Unsupported CDM specified. Supported CDM versions are \"5.3\" and \"5.4\".")
   }
 
-  supportedSyntheaVersions <- c("2.7.0", "3.0.0", "3.1.0", "3.2.0")
+  supportedSyntheaVersions <- c("2.7.0", "3.0.0", "3.1.0", "3.2.0", "3.3.0")
 
   if (!(syntheaVersion %in% supportedSyntheaVersions))
     stop(
-      "Invalid Synthea version specified. Currently \"2.7.0\", \"3.0.0\",\"3.1.0\", and \"3.2.0\" are supported."
+      "Invalid Synthea version specified. Currently \"2.7.0\", \"3.0.0\",\"3.1.0\",\"3.2.0\", and \"3.3.0\" are supported."
     )
 
   if (createIndices) {
@@ -177,7 +177,8 @@ LoadEventTables <- function(connectionDetails,
     packageName = "ETLSyntheaBuilder",
     dbms = connectionDetails$dbms,
     cdm_schema = cdmSchema,
-    synthea_schema = syntheaSchema
+    synthea_schema = syntheaSchema,
+    synthea_version = syntheaVersion
   )
   runStep(sql, fileQuery)
 
@@ -299,7 +300,7 @@ LoadEventTables <- function(connectionDetails,
   # cost
   if (syntheaVersion == "2.7.0")
     fileQuery <- "insert_cost_v270.sql"
-  else if (syntheaVersion %in% c("3.0.0", "3.1.0", "3.2.0"))
+  else if (syntheaVersion %in% c("3.0.0", "3.1.0", "3.2.0", "3.3.0"))
     fileQuery <- "insert_cost_v300.sql"
 
   sql <- SqlRender::loadRenderTranslateSql(

diff --git a/R/LoadVocabFromCsv.r b/R/LoadVocabFromCsv.r
@@ -9,10 +9,11 @@
 #'                                     \code{DatabaseConnector} package.
 #' @param cdmSchema  The name of the database schema that will contain the Vocabulary (and CDM)
 #'                                     tables.  Requires read and write permissions to this database. On SQL
-#'                                     Server, this should specifiy both the database and the schema,
+#'                                     Server, this should specify both the database and the schema,
 #'                                     so for example 'cdm_instance.dbo'.
 #' @param vocabFileLoc     The location of the vocabulary csv files.
 #' @param bulkLoad       Boolean flag indicating whether or not to use bulk loading (if possible).  Default is FALSE.
+#' @param delimiter       Parameter to specify the delimiter of the csv file. By default the function expects the file to be tab delimited ("\\t") based on the export from Athena.
 #'
 #'@export
 
@@ -21,7 +22,8 @@ LoadVocabFromCsv <-
   function(connectionDetails,
            cdmSchema,
            vocabFileLoc,
-           bulkLoad = FALSE)
+           bulkLoad = FALSE,
+           delimiter = "\t")
   {
     csvList <- c(
       "concept.csv",
@@ -50,7 +52,7 @@ LoadVocabFromCsv <-
           file = paste0(vocabFileLoc, "/", csv),
           stringsAsFactors = FALSE,
           header = TRUE,
-          sep = "\t",
+          sep = delimiter,
           na.strings = ""
         )
 

diff --git a/R/createExtraIndices.R b/R/createExtraIndices.R
@@ -8,13 +8,13 @@
 #'                  Server, this should specify both the database and the schema, so for example 'cdm_instance.dbo'.
 #' @param syntheaSchema The name of the Synthea database schema. Requires read and write permissions to this schema. On SQL
 #'                      Server, this should specify both the database and the schema, so for example 'synthea.dbo'.
-#' @param syntheaVersion Your Synthea version. Currently "2.7.0", "3.0.0", "3.1.0" and "3.2.0" are supported.
+#' @param syntheaVersion Your Synthea version. Currently "2.7.0", "3.0.0", "3.1.0", "3.2.0" and "3.3.0" are supported.
 #' @param outputFolder Location of the SQL scripts if sqlOnly = TRUE. Default is NULL.
 #' @param sqlOnly A boolean that determines whether to create the indices or generate a SQL scripts. Default is FALSE.
 #'
 #' @details This function creates indices which have been found to speed up certain long-running INSERT queries in LoadEventTables,
 #'          for some users. Indices are created on the intermediate vocabulary mapping tables; the person & provider CDM tables;
-#'          and the claims_transactions Synthea table (in Synthea 3.0.0, 3.1.0 and 3.2.0) .
+#'          and the claims_transactions Synthea table (in Synthea 3.0.0, 3.1.0, 3.2.0 and 3.3.0) .
 #'
 #' @importFrom utils head
 #'

diff --git a/README.md b/README.md
@@ -16,7 +16,7 @@ Follow the steps on the [synthea wiki](https://github.com/synthetichealth/synthe
  # Valid CDM versions are determined by executing CommonDataModel::listSupportedVersions().
  # The strings representing supported CDM versions are currently "5.3" and "5.4". 
  # The Synthea version we use in this example is 2.7.0.
- # However, at this time we also support 3.0.0, 3.1.0 and 3.2.0.
+ # However, at this time we also support 3.0.0, 3.1.0, 3.2.0 and 3.3.0.
  # Please note that Synthea's MASTER branch is always active and this package will be updated to support
  # future versions as possible.
  # The schema to load the Synthea tables is called "native".

diff --git a/docs/404.html b/docs/404.html
diff --git a/docs/articles/care_site.html b/docs/articles/care_site.html
diff --git a/docs/articles/condition_occurrence.html b/docs/articles/condition_occurrence.html
diff --git a/docs/articles/cost.html b/docs/articles/cost.html
diff --git a/docs/articles/death.html b/docs/articles/death.html
diff --git a/docs/articles/device_exposure.html b/docs/articles/device_exposure.html
diff --git a/docs/articles/drug_exposure.html b/docs/articles/drug_exposure.html
diff --git a/docs/articles/index.html b/docs/articles/index.html
diff --git a/docs/articles/location.html b/docs/articles/location.html
diff --git a/docs/articles/measurement.html b/docs/articles/measurement.html
diff --git a/docs/articles/observation.html b/docs/articles/observation.html
diff --git a/docs/articles/observation_period.html b/docs/articles/observation_period.html
diff --git a/docs/articles/payer_plan_period.html b/docs/articles/payer_plan_period.html
diff --git a/docs/articles/person.html b/docs/articles/person.html
diff --git a/docs/articles/procedure_occurrence.html b/docs/articles/procedure_occurrence.html
diff --git a/docs/articles/provider.html b/docs/articles/provider.html
diff --git a/docs/articles/visit_detail.html b/docs/articles/visit_detail.html
diff --git a/docs/articles/visit_occurrence.html b/docs/articles/visit_occurrence.html
diff --git a/docs/authors.html b/docs/authors.html