waveform POC-2; bugfixes

OHDSI · Feb 1, 2021 · beb3602 · beb3602
1 parent 0c4fe81
commit beb3602
Show file tree

Hide file tree

Showing 17 changed files with 1,476 additions and 54 deletions.
diff --git a/README.md b/README.md
@@ -31,6 +31,12 @@
 ### Change Log (latest first) ###
 
 
+**2021-02-01**
+
+* Waveforms POC-2 (load from folders tree and csv files)
+* Bugfixes
+
+
 **2021-01-25**
 
 * Mapping improvement

diff --git a/custom_mapping_csv/gcpt_meas_lab_loinc.csv b/custom_mapping_csv/gcpt_meas_lab_loinc.csv
@@ -82,7 +82,6 @@
 "Ventilator",2000001081,"mimiciv_meas_lab_loinc","Measurement","Lab Test",,"Ventilator","1970-01-01","2099-12-31",,3004921,"Maps to","Mapped from","1970-01-01","2099-12-31",
 "Intubated",2000001082,"mimiciv_meas_lab_loinc","Measurement","Lab Test",,"Intubated","1970-01-01","2099-12-31",,45884415,"Maps to","Mapped from","1970-01-01","2099-12-31",
 "Alveolar-arterial Gradient",2000001083,"mimiciv_meas_lab_loinc","Measurement","Lab Test",,"Alveolar-arterial Gradient","1970-01-01","2099-12-31",,3007913,"Maps to","Mapped from","1970-01-01","2099-12-31",
-"Calculated Bicarbonate, Whole Blood",2000001084,"mimiciv_meas_lab_loinc","Measurement","Lab Test",,"Calculated Bicarbonate, Whole Blood","1970-01-01","2099-12-31",,0,"Maps to","Mapped from","1970-01-01","2099-12-31",
 "Chloride, Whole Blood",2000001085,"mimiciv_meas_lab_loinc","Measurement","Lab Test",,"Chloride, Whole Blood","1970-01-01","2099-12-31",,3018572,"Maps to","Mapped from","1970-01-01","2099-12-31",
 "Free Calcium",2000001086,"mimiciv_meas_lab_loinc","Measurement","Lab Test",,"Free Calcium","1970-01-01","2099-12-31",,3021119,"Maps to","Mapped from","1970-01-01","2099-12-31",
 "Glucose",2000001087,"mimiciv_meas_lab_loinc","Measurement","Lab Test",,"Glucose","1970-01-01","2099-12-31",,3000483,"Maps to","Mapped from","1970-01-01","2099-12-31",
@@ -92,7 +91,6 @@
 "Oxygen",2000001091,"mimiciv_meas_lab_loinc","Measurement","Lab Test",,"Oxygen","1970-01-01","2099-12-31",,3024882,"Maps to","Mapped from","1970-01-01","2099-12-31",
 "Oxygen Saturation",2000001092,"mimiciv_meas_lab_loinc","Measurement","Lab Test",,"Oxygen Saturation","1970-01-01","2099-12-31",,3013502,"Maps to","Mapped from","1970-01-01","2099-12-31",
 "PEEP",2000001093,"mimiciv_meas_lab_loinc","Measurement","Lab Test",,"PEEP","1970-01-01","2099-12-31",,3022875,"Maps to","Mapped from","1970-01-01","2099-12-31",
-"Calculated Bicarbonate, Whole Blood",2000001094,"mimiciv_meas_lab_loinc","Measurement","Lab Test",,"Calculated Bicarbonate, Whole Blood","1970-01-01","2099-12-31",,3005456,"Maps to","Mapped from","1970-01-01","2099-12-31",
 "Sodium, Whole Blood",2000001095,"mimiciv_meas_lab_loinc","Measurement","Lab Test",,"Sodium, Whole Blood","1970-01-01","2099-12-31",,3000285,"Maps to","Mapped from","1970-01-01","2099-12-31",
 "Tidal Volume",2000001096,"mimiciv_meas_lab_loinc","Measurement","Lab Test",,"Tidal Volume","1970-01-01","2099-12-31",,3012410,"Maps to","Mapped from","1970-01-01","2099-12-31",
 "Barbiturate Screen",2000001097,"mimiciv_meas_lab_loinc","Measurement","Lab Test",,"Barbiturate Screen","1970-01-01","2099-12-31",,3003132,"Maps to","Mapped from","1970-01-01","2099-12-31",
@@ -105,7 +103,6 @@
 "Osmolality, Measured",2000001104,"mimiciv_meas_lab_loinc","Measurement","Lab Test",,"Osmolality, Measured","1970-01-01","2099-12-31",,3008295,"Maps to","Mapped from","1970-01-01","2099-12-31",
 "Protein Electrophoresis",2000001105,"mimiciv_meas_lab_loinc","Measurement","Lab Test",,"Protein Electrophoresis","1970-01-01","2099-12-31",,3004588,"Maps to","Mapped from","1970-01-01","2099-12-31",
 "Tricyclic Antidepressant Screen",2000001106,"mimiciv_meas_lab_loinc","Measurement","Lab Test",,"Tricyclic Antidepressant Screen","1970-01-01","2099-12-31",,3025478,"Maps to","Mapped from","1970-01-01","2099-12-31",
-"Pleural fluid lactate dehydrogenase measurement",2000001107,"mimiciv_meas_lab_loinc","Measurement","Lab Test",,"Lactate Dehydrogenase, Pleural","1970-01-01","2099-12-31",,4135658,"Maps to","Mapped from","1970-01-01","2099-12-31",
 "Total Protein, Pleural",2000001108,"mimiciv_meas_lab_loinc","Measurement","Lab Test",,"Total Protein, Pleural","1970-01-01","2099-12-31",,3003434,"Maps to","Mapped from","1970-01-01","2099-12-31",
 "Amphetamine Screen, Urine",2000001109,"mimiciv_meas_lab_loinc","Measurement","Lab Test",,"Amphetamine Screen, Urine","1970-01-01","2099-12-31",,3027944,"Maps to","Mapped from","1970-01-01","2099-12-31",
 "Benzodiazepine Screen, Urine",2000001110,"mimiciv_meas_lab_loinc","Measurement","Lab Test",,"Benzodiazepine Screen, Urine","1970-01-01","2099-12-31",,3000764,"Maps to","Mapped from","1970-01-01","2099-12-31",
@@ -116,18 +113,14 @@
 "Opiate Screen, Urine",2000001115,"mimiciv_meas_lab_loinc","Measurement","Lab Test",,"Opiate Screen, Urine","1970-01-01","2099-12-31",,3027008,"Maps to","Mapped from","1970-01-01","2099-12-31",
 "Osmolality, Urine",2000001116,"mimiciv_meas_lab_loinc","Measurement","Lab Test",,"Osmolality, Urine","1970-01-01","2099-12-31",,3026782,"Maps to","Mapped from","1970-01-01","2099-12-31",
 "Urea Nitrogen, Urine",2000001117,"mimiciv_meas_lab_loinc","Measurement","Lab Test",,"Urea Nitrogen, Urine","1970-01-01","2099-12-31",,3011965,"Maps to","Mapped from","1970-01-01","2099-12-31",
-"Lymphocytes",2000001118,"mimiciv_meas_lab_loinc","Measurement","Lab Test",,"Lymphocytes","1970-01-01","2099-12-31",,3004437,"Maps to","Mapped from","1970-01-01","2099-12-31",
 "Macrophage",2000001119,"mimiciv_meas_lab_loinc","Measurement","Lab Test",,"Macrophage","1970-01-01","2099-12-31",,3041635,"Maps to","Mapped from","1970-01-01","2099-12-31",
 "Monocytes",2000001120,"mimiciv_meas_lab_loinc","Measurement","Lab Test",,"Monocytes","1970-01-01","2099-12-31",,3033483,"Maps to","Mapped from","1970-01-01","2099-12-31",
-"Polys",2000001121,"mimiciv_meas_lab_loinc","Measurement","Lab Test",,"Polys","1970-01-01","2099-12-31",,3008838,"Maps to","Mapped from","1970-01-01","2099-12-31",
 "RBC, Ascites",2000001122,"mimiciv_meas_lab_loinc","Measurement","Lab Test",,"RBC, Ascites","1970-01-01","2099-12-31",,3009613,"Maps to","Mapped from","1970-01-01","2099-12-31",
 "Fibrinogen, Functional",2000001123,"mimiciv_meas_lab_loinc","Measurement","Lab Test",,"Fibrinogen, Functional","1970-01-01","2099-12-31",,3016407,"Maps to","Mapped from","1970-01-01","2099-12-31",
 "Granulocyte Count",2000001124,"mimiciv_meas_lab_loinc","Measurement","Lab Test",,"Granulocyte Count","1970-01-01","2099-12-31",,3035715,"Maps to","Mapped from","1970-01-01","2099-12-31",
 "Reticulocyte Count, Absolute",2000001125,"mimiciv_meas_lab_loinc","Measurement","Lab Test",,"Reticulocyte Count, Absolute","1970-01-01","2099-12-31",,3023520,"Maps to","Mapped from","1970-01-01","2099-12-31",
 "Reticulocyte Count, Automated",2000001126,"mimiciv_meas_lab_loinc","Measurement","Lab Test",,"Reticulocyte Count, Automated","1970-01-01","2099-12-31",,3007124,"Maps to","Mapped from","1970-01-01","2099-12-31",
-"Lymphocytes",2000001127,"mimiciv_meas_lab_loinc","Measurement","Lab Test",,"Lymphocytes","1970-01-01","2099-12-31",,3005532,"Maps to","Mapped from","1970-01-01","2099-12-31",
 "Monos",2000001128,"mimiciv_meas_lab_loinc","Measurement","Lab Test",,"Monos","1970-01-01","2099-12-31",,3043387,"Maps to","Mapped from","1970-01-01","2099-12-31",
-"Polys",2000001129,"mimiciv_meas_lab_loinc","Measurement","Lab Test",,"Polys","1970-01-01","2099-12-31",,3026051,"Maps to","Mapped from","1970-01-01","2099-12-31",
 "Blood",2000001130,"mimiciv_meas_lab_loinc","Measurement","Lab Test",,"Blood","1970-01-01","2099-12-31",,3011397,"Maps to","Mapped from","1970-01-01","2099-12-31",
 "Eosinophils",2000001131,"mimiciv_meas_lab_loinc","Measurement","Lab Test",,"Eosinophils","1970-01-01","2099-12-31",,3037579,"Maps to","Mapped from","1970-01-01","2099-12-31",
 "Transitional Epithelial Cells",2000001132,"mimiciv_meas_lab_loinc","Measurement","Lab Test",,"Transitional Epithelial Cells","1970-01-01","2099-12-31",,3035851,"Maps to","Mapped from","1970-01-01","2099-12-31",
@@ -155,12 +148,10 @@
 "Cytomegalovirus DNA [Units/volume] (viral load) in Unspecified specimen by Probe with signal amplification",2000001154,"mimiciv_meas_lab_loinc","Measurement","Lab Test",,"Cytomegalovirus Viral Load","1970-01-01","2099-12-31",,3022489,"Maps to","Mapped from","1970-01-01","2099-12-31",
 "Hepatitis B virus core IgM Ab [Units/volume] in Serum",2000001155,"mimiciv_meas_lab_loinc","Measurement","Lab Test",,"Hepatitis B Core Antibody, IgM","1970-01-01","2099-12-31",,3018806,"Maps to","Mapped from","1970-01-01","2099-12-31",
 "Cholesterol in LDL [Moles/volume] in Serum or Plasma",2000001156,"mimiciv_meas_lab_loinc","Measurement","Lab Test",,"Cholesterol, LDL, Measured","1970-01-01","2099-12-31",,3001308,"Maps to","Mapped from","1970-01-01","2099-12-31",
-"Lactate dehydrogenase [Enzymatic activity/volume] in Pleural fluid by Pyruvate to lactate reaction",2000001157,"mimiciv_meas_lab_loinc","Measurement","Lab Test",,"Lactate Dehydrogenase, Pleural","1970-01-01","2099-12-31",,40763081,"Maps to","Mapped from","1970-01-01","2099-12-31",
 "Erythrocytes [#/volume] in Pleural fluid",2000001158,"mimiciv_meas_lab_loinc","Measurement","Lab Test",,"RBC, Pleural","1970-01-01","2099-12-31",,3028308,"Maps to","Mapped from","1970-01-01","2099-12-31",
 "Glucose [Mass/volume] in Pleural fluid",2000001159,"mimiciv_meas_lab_loinc","Measurement","Lab Test",,"Glucose, Pleural","1970-01-01","2099-12-31",,3003403,"Maps to","Mapped from","1970-01-01","2099-12-31",
 "Lymphocytes [#/volume] in Blood",2000001160,"mimiciv_meas_lab_loinc","Measurement","Lab Test",,"Absolute Lymphocyte Count","1970-01-01","2099-12-31",,3019198,"Maps to","Mapped from","1970-01-01","2099-12-31",
 "Erythrocyte shape [Morphology] in Blood",2000001161,"mimiciv_meas_lab_loinc","Measurement","Lab Test",,"RBC Morphology","1970-01-01","2099-12-31",,3014029,"Maps to","Mapped from","1970-01-01","2099-12-31",
-"Glomerular filtration rate/1.73 sq M.predicted [Volume Rate/Area] in Serum, Plasma or Blood by Creatinine-based formula (MDRD)",2000001162,"mimiciv_meas_lab_loinc","Measurement","Lab Test",,"Estimated GFR (MDRD equation)","1970-01-01","2099-12-31",,46236952,"Maps to","Mapped from","1970-01-01","2099-12-31",
 "Potassium [Moles/volume] in Blood",2000001163,"mimiciv_meas_lab_loinc","Measurement","Lab Test",,"Potassium, Whole Blood","1970-01-01","2099-12-31",,3005456,"Maps to","Mapped from","1970-01-01","2099-12-31",
 "Immature granulocytes [#/volume] in Blood",2000001164,"mimiciv_meas_lab_loinc","Measurement","Lab Test",,"Immature Granulocytes","1970-01-01","2099-12-31",,3040168,"Maps to","Mapped from","1970-01-01","2099-12-31",
 "Protein [Mass/volume] in Peritoneal fluid",2000001165,"mimiciv_meas_lab_loinc","Measurement","Lab Test",,"Total Protein, Ascites","1970-01-01","2099-12-31",,3002331,"Maps to","Mapped from","1970-01-01","2099-12-31",

diff --git a/etl/etl/cdm_measurement.sql b/etl/etl/cdm_measurement.sql
@@ -24,7 +24,6 @@
 -- src_labevents.value: 
 --      investigate if there are formatted values with thousand separators,
 --      and if we need to use more complicated parsing.
---      see `@etl_project`.@etl_dataset.an_labevents_full
 -- -------------------------------------------------------------------
 
 
@@ -261,7 +260,7 @@ SELECT
     44807951                            AS measurement_type_concept_id, -- ECG - electrocardiograph -- ???
     CAST(NULL AS INT64)                     AS operator_concept_id,
     src.value_as_number                     AS value_as_number,
-    CAST(NULL AS INT64)                     AS value_as_concept_id,
+    CAST(NULL AS INT64)                     AS value_as_concept_id, -- to add values
     src.unit_concept_id                     AS unit_concept_id,
     CAST(NULL AS FLOAT64)                   AS range_low,
     CAST(NULL AS FLOAT64)                   AS range_high,

diff --git a/etl/etl/cdm_observation.sql b/etl/etl/cdm_observation.sql
@@ -67,7 +67,9 @@ SELECT
     src.type_concept_id                         AS observation_type_concept_id,
     CAST(NULL AS FLOAT64)                       AS value_as_number,
     src.value_as_string                         AS value_as_string,
-    src.value_as_concept_id                     AS value_as_concept_id,
+    IF(src.value_as_string IS NOT NULL,
+        COALESCE(src.value_as_concept_id, 0),
+        NULL)                                   AS value_as_concept_id,
     CAST(NULL AS INT64)                         AS qualifier_concept_id,
     CAST(NULL AS INT64)                         AS unit_concept_id,
     CAST(NULL AS INT64)                         AS provider_id,
@@ -110,7 +112,9 @@ SELECT
     32817                                       AS observation_type_concept_id, -- EHR  Type Concept    Type Concept
     src.value_as_number                         AS value_as_number,
     src.value_source_value                      AS value_as_string,
-    src.value_as_concept_id                     AS value_as_concept_id,
+    IF(src.value_source_value IS NOT NULL,
+        COALESCE(src.value_as_concept_id, 0),
+        NULL)                                   AS value_as_concept_id,
     CAST(NULL AS INT64)                         AS qualifier_concept_id,
     src.unit_concept_id                         AS unit_concept_id,
     CAST(NULL AS INT64)                         AS provider_id,

diff --git a/etl/etl/cdm_procedure_occurrence.sql b/etl/etl/cdm_procedure_occurrence.sql
@@ -89,7 +89,7 @@ WHERE
 
 -- -------------------------------------------------------------------
 -- Rule 5
--- lk_observation_mapped
+-- lk_observation_mapped, possible DRG codes
 -- -------------------------------------------------------------------
 
 INSERT INTO `@etl_project`.@etl_dataset.cdm_procedure_occurrence

diff --git a/etl/etl/lk_drug.sql b/etl/etl/lk_drug.sql
@@ -53,6 +53,7 @@ SELECT
         src.drug, '')
         || ' ' || COALESCE(src.prod_strength, '')               AS gcpt_source_code,
     'mimiciv_drug_ndc'                                          AS gcpt_source_vocabulary, -- source_code = label
+    src.pharmacy_id                                             AS pharmacy_id,
     -- 
     'prescriptions'                 AS unit_id,
     src.load_table_id               AS load_table_id,
@@ -173,6 +174,7 @@ SELECT
     src.route_source_code                                               AS route_source_code,
     src.dose_unit_source_code                       AS dose_unit_source_code,
     src.form_val_disp                               AS quantity_source_value,
+    src.pharmacy_id                                 AS pharmacy_id, -- to investigate pharmacy.medication
     -- 
     CONCAT('drug.', src.unit_id)    AS unit_id,
     src.load_table_id               AS load_table_id,

diff --git a/etl/etl/lk_meas_chartevents.sql b/etl/etl/lk_meas_chartevents.sql
@@ -138,7 +138,7 @@ LEFT JOIN
     `@etl_project`.@etl_dataset.lk_chartevents_concept c_value -- values for main
         ON c_value.source_code = src.value
         AND c_value.source_vocabulary_id = 'mimiciv_meas_chartevents_value'
-        AND c_value.target_domain_id = 'Observation'
+        AND c_value.target_domain_id = 'Meas Value'
 LEFT JOIN 
     `@etl_project`.@etl_dataset.lk_meas_unit_concept uc
         ON uc.source_code = src.valueuom

diff --git a/etl/etl/lk_meas_waveform.sql b/etl/etl/lk_meas_waveform.sql
@@ -7,7 +7,7 @@
 -- Populate lookups for cdm_measurement table
 -- Rule 10 waveforms
 -- Dependencies: run after 
---      st_waveform.sql,
+--      st_waveform_poc2.sql,
 --      lk_meas_unit_concept
 -- -------------------------------------------------------------------
 
@@ -100,7 +100,10 @@ SELECT
     FARM_FINGERPRINT(GENERATE_UUID())       AS measurement_id,
     wh.subject_id                           AS subject_id,
     hadm.hadm_id                            AS hadm_id,     -- get hadm_id by datetime period
-    src.reference_id                        AS reference_id,
+    CONCAT(
+        src.reference_id, '.', src.segment_name,
+        '.', src.source_code
+    )                                       AS reference_id, -- add segment name and source code to make the field unique
     COALESCE(vc2.concept_id, 0)             AS target_concept_id,
     COALESCE(vc2.domain_id, 'Measurement')  AS target_domain_id,
     src.mx_datetime                         AS start_datetime,
@@ -129,7 +132,7 @@ LEFT JOIN
 LEFT JOIN
     `@etl_project`.@etl_dataset.voc_concept vc1
         ON vc1.concept_code = src.source_code
-        AND vc1.vocabulary_id = 'mimiciv_meas_waveform_code'
+        AND vc1.vocabulary_id = 'mimiciv_meas_wf'
             -- supposing that the standard mapping is supplemented with custom concepts for waveform specific values
 LEFT JOIN
     `@etl_project`.@etl_dataset.voc_concept_relationship vr

diff --git a/etl/staging/st_waveform.sql → etl/staging/st_waveform_poc1.sql b/etl/staging/st_waveform.sql → etl/staging/st_waveform_poc1.sql