Terminology Versioning and ICD 10 CM Update (#461)

* created post hook for all seed files * tested all seed files; cleaned terminology files out * removed content from value set seeds * removed content from value set seeds * added data types to seed files to prevent load errors * added data types to seed files to prevent load errors * fixed empty row bug * updated icd10cm with new headers and header codes; fixed reference to old description column * updated s3 path for provider.csv * updated docs and version number * updated docs and version number * added data types causing errors * update column length to fix redshift error; removed blank line for BQ error * fixed data type mismatch BQ * edited syntax for varchar(max); fixed data types * edited syntax for varchar(max); fixed data types * edited syntax for setting column length
tuva-health · May 10, 2024 · 91cf3bb · 91cf3bb
1 parent b2c7435
commit 91cf3bb
Show file tree

Hide file tree

Showing 68 changed files with 597 additions and 70,814 deletions.
diff --git a/dbt_project.yml b/dbt_project.yml
diff --git a/docs/catalog.json b/docs/catalog.json
diff --git a/docs/manifest.json b/docs/manifest.json
diff --git a/docs/run_results.json b/docs/run_results.json
diff --git a/models/claims_preprocessing/acute_inpatient/final/acute_inpatient__summary.sql b/models/claims_preprocessing/acute_inpatient/final/acute_inpatient__summary.sql
@@ -98,7 +98,7 @@ select
 , e.race
 , c.diagnosis_code_type as primary_diagnosis_code_type
 , c.diagnosis_code_1 as primary_diagnosis_code
-, coalesce(icd10cm.description, icd9cm.long_description) as primary_diagnosis_description
+, coalesce(icd10cm.long_description, icd9cm.long_description) as primary_diagnosis_description
 , f.facility_npi
 , f.provider_first_name
 , f.provider_last_name

diff --git a/models/claims_preprocessing/emergency_department/final/emergency_department__summary.sql b/models/claims_preprocessing/emergency_department/final/emergency_department__summary.sql
@@ -1,150 +1,151 @@
-{{ config(
-     enabled = var('claims_preprocessing_enabled',var('claims_enabled',var('tuva_marts_enabled',False))) | as_bool
-   )
-}}
-
-with distinct_encounters as (
-select distinct
-  a.encounter_id
-, a.patient_id
-, b.encounter_start_date
-, b.encounter_end_date
-from {{ ref('emergency_department__int_encounter_id') }} a
-inner join {{ ref('emergency_department__int_encounter_start_and_end_dates') }} b
-  on a.encounter_id = b.encounter_id
-)
-
-, institutional_claim_details as (
-    select
-        b.encounter_id
-        , first.diagnosis_code_1
-        , first.diagnosis_code_type
-        , first.facility_npi as facility_npi
-        , first.ms_drg_code as ms_drg_code
-        , first.apr_drg_code as apr_drg_code
-        , first.admit_source_code as admit_source_code
-        , first.admit_type_code as admit_type_code
-        , last.discharge_disposition_code as discharge_disposition_code
-        , sum(paid_amount) as inst_paid_amount
-        , sum(allowed_amount) as inst_allowed_amount
-        , sum(charge_amount) as inst_charge_amount
-        , max(data_source) as data_source
-    from {{ ref('medical_claim') }} a
-    inner join {{ ref('emergency_department__int_encounter_id') }} b
-        on a.claim_id = b.claim_id
-        and a.claim_line_number = b.claim_line_number
-        and a.claim_type = 'institutional'
-    inner join {{ ref('emergency_department__int_first_claim_values') }} first
-        on b.encounter_id = first.encounter_id
-        and first.claim_row = 1
-    inner join {{ ref('emergency_department__int_last_claim_values') }} last
-        on b.encounter_id = last.encounter_id
-        and last.claim_row = 1
-    group by
-    b.encounter_id
-    , first.diagnosis_code_1
-    , first.diagnosis_code_type
-    , first.facility_npi
-    , first.ms_drg_code
-    , first.apr_drg_code
-    , first.admit_source_code
-    , first.admit_type_code
-    , last.discharge_disposition_code
-)
-
-, professional_claim_details as (
-select
-  b.encounter_id
-, sum(paid_amount) as prof_paid_amount
-, sum(allowed_amount) as prof_allowed_amount
-, sum(charge_amount) as prof_charge_amount
-from {{ ref('medical_claim') }} a
-inner join {{ ref('emergency_department__int_encounter_id') }} b
-  on a.claim_id = b.claim_id
-  and a.claim_line_number = b.claim_line_number
-  and a.claim_type = 'professional'
-group by 1
-)
-
-, patient as (
-select distinct
-  patient_id
-, birth_date
-, gender
-, race
-from {{ ref('eligibility') }}
-)
-
-, provider as (
-select
-  a.encounter_id
-, max(a.facility_npi) as facility_npi
-, b.provider_first_name
-, b.provider_last_name
-, count(distinct facility_npi) as npi_count
-from {{ ref('emergency_department__int_institutional_encounter_id') }} a
-left join {{ ref('terminology__provider') }} b
-  on a.facility_npi = b.npi
-group by 1,3,4
-)
-
-select
-    a.encounter_id
-    , a.encounter_start_date
-    , a.encounter_end_date
-    , a.patient_id
-    , {{ dbt.datediff("birth_date","encounter_end_date","day")}}/365 as admit_age
-    , e.gender
-    , e.race
-    , c.diagnosis_code_type as primary_diagnosis_code_type
-    , c.diagnosis_code_1 as primary_diagnosis_code
-    , coalesce(icd10cm.description, icd9cm.long_description) as primary_diagnosis_description
-    , f.facility_npi
-    , f.provider_first_name
-    , f.provider_last_name
-    , c.ms_drg_code
-    , j.ms_drg_description
-    , j.medical_surgical
-    , c.apr_drg_code
-    , k.apr_drg_description
-    , c.admit_source_code
-    , h.admit_source_description
-    , c.admit_type_code
-    , i.admit_type_description
-    , c.discharge_disposition_code
-    , g.discharge_disposition_description
-    , c.inst_paid_amount + coalesce(d.prof_paid_amount,0) as total_paid_amount
-    , c.inst_allowed_amount + coalesce(d.prof_allowed_amount,0) as total_allowed_amount
-    , c.inst_charge_amount + coalesce(d.prof_charge_amount,0) as total_charge_amount
-    , {{ dbt.datediff("a.encounter_start_date","a.encounter_end_date","day") }} as length_of_stay
-    , case
-        when c.discharge_disposition_code = '20' then 1
-        else 0
-    end mortality_flag
-    , data_source
-    , '{{ var('tuva_last_run')}}' as tuva_last_run
-from distinct_encounters a
-left join institutional_claim_details c
-  on a.encounter_id = c.encounter_id
-left join professional_claim_details d
-  on a.encounter_id = d.encounter_id
-left join patient e
-  on a.patient_id = e.patient_id
-left join provider f
-  on a.encounter_id = f.encounter_id
-left join {{ ref('terminology__discharge_disposition') }} g
-  on c.discharge_disposition_code = g.discharge_disposition_code
-left join {{ ref('terminology__admit_source') }} h
-  on c.admit_source_code = h.admit_source_code
-left join {{ ref('terminology__admit_type') }} i
-  on c.admit_type_code = i.admit_type_code
-left join {{ ref('terminology__ms_drg') }} j
-  on c.ms_drg_code = j.ms_drg_code
-left join {{ ref('terminology__apr_drg') }} k
-  on c.apr_drg_code = k.apr_drg_code
-left join {{ ref('terminology__icd_10_cm')}} icd10cm
-  on c.diagnosis_code_1 = icd10cm.icd_10_cm
-  and c.diagnosis_code_type = 'icd-10-cm'
-left join {{ ref('terminology__icd_9_cm')}} icd9cm
-  on c.diagnosis_code_1 = icd9cm.icd_9_cm
+{{ config(
+     enabled = var('claims_preprocessing_enabled',var('claims_enabled',var('tuva_marts_enabled',False)))
+ | as_bool
+   )
+}}
+
+with distinct_encounters as (
+select distinct
+  a.encounter_id
+, a.patient_id
+, b.encounter_start_date
+, b.encounter_end_date
+from {{ ref('emergency_department__int_encounter_id') }} a
+inner join {{ ref('emergency_department__int_encounter_start_and_end_dates') }} b
+  on a.encounter_id = b.encounter_id
+)
+
+, institutional_claim_details as (
+    select
+        b.encounter_id
+        , first.diagnosis_code_1
+        , first.diagnosis_code_type
+        , first.facility_npi as facility_npi
+        , first.ms_drg_code as ms_drg_code
+        , first.apr_drg_code as apr_drg_code
+        , first.admit_source_code as admit_source_code
+        , first.admit_type_code as admit_type_code
+        , last.discharge_disposition_code as discharge_disposition_code
+        , sum(paid_amount) as inst_paid_amount
+        , sum(allowed_amount) as inst_allowed_amount
+        , sum(charge_amount) as inst_charge_amount
+        , max(data_source) as data_source
+    from {{ ref('medical_claim') }} a
+    inner join {{ ref('emergency_department__int_encounter_id') }} b
+        on a.claim_id = b.claim_id
+        and a.claim_line_number = b.claim_line_number
+        and a.claim_type = 'institutional'
+    inner join {{ ref('emergency_department__int_first_claim_values') }} first
+        on b.encounter_id = first.encounter_id
+        and first.claim_row = 1
+    inner join {{ ref('emergency_department__int_last_claim_values') }} last
+        on b.encounter_id = last.encounter_id
+        and last.claim_row = 1
+    group by
+    b.encounter_id
+    , first.diagnosis_code_1
+    , first.diagnosis_code_type
+    , first.facility_npi
+    , first.ms_drg_code
+    , first.apr_drg_code
+    , first.admit_source_code
+    , first.admit_type_code
+    , last.discharge_disposition_code
+)
+
+, professional_claim_details as (
+select
+  b.encounter_id
+, sum(paid_amount) as prof_paid_amount
+, sum(allowed_amount) as prof_allowed_amount
+, sum(charge_amount) as prof_charge_amount
+from {{ ref('medical_claim') }} a
+inner join {{ ref('emergency_department__int_encounter_id') }} b
+  on a.claim_id = b.claim_id
+  and a.claim_line_number = b.claim_line_number
+  and a.claim_type = 'professional'
+group by 1
+)
+
+, patient as (
+select distinct
+  patient_id
+, birth_date
+, gender
+, race
+from {{ ref('eligibility') }}
+)
+
+, provider as (
+select
+  a.encounter_id
+, max(a.facility_npi) as facility_npi
+, b.provider_first_name
+, b.provider_last_name
+, count(distinct facility_npi) as npi_count
+from {{ ref('emergency_department__int_institutional_encounter_id') }} a
+left join {{ ref('terminology__provider') }} b
+  on a.facility_npi = b.npi
+group by 1,3,4
+)
+
+select
+    a.encounter_id
+    , a.encounter_start_date
+    , a.encounter_end_date
+    , a.patient_id
+    , {{ dbt.datediff("birth_date","encounter_end_date","day")}}/365 as admit_age
+    , e.gender
+    , e.race
+    , c.diagnosis_code_type as primary_diagnosis_code_type
+    , c.diagnosis_code_1 as primary_diagnosis_code
+    , coalesce(icd10cm.long_description, icd9cm.long_description) as primary_diagnosis_description
+    , f.facility_npi
+    , f.provider_first_name
+    , f.provider_last_name
+    , c.ms_drg_code
+    , j.ms_drg_description
+    , j.medical_surgical
+    , c.apr_drg_code
+    , k.apr_drg_description
+    , c.admit_source_code
+    , h.admit_source_description
+    , c.admit_type_code
+    , i.admit_type_description
+    , c.discharge_disposition_code
+    , g.discharge_disposition_description
+    , c.inst_paid_amount + coalesce(d.prof_paid_amount,0) as total_paid_amount
+    , c.inst_allowed_amount + coalesce(d.prof_allowed_amount,0) as total_allowed_amount
+    , c.inst_charge_amount + coalesce(d.prof_charge_amount,0) as total_charge_amount
+    , {{ dbt.datediff("a.encounter_start_date","a.encounter_end_date","day") }} as length_of_stay
+    , case
+        when c.discharge_disposition_code = '20' then 1
+        else 0
+    end mortality_flag
+    , data_source
+    , '{{ var('tuva_last_run')}}' as tuva_last_run
+from distinct_encounters a
+left join institutional_claim_details c
+  on a.encounter_id = c.encounter_id
+left join professional_claim_details d
+  on a.encounter_id = d.encounter_id
+left join patient e
+  on a.patient_id = e.patient_id
+left join provider f
+  on a.encounter_id = f.encounter_id
+left join {{ ref('terminology__discharge_disposition') }} g
+  on c.discharge_disposition_code = g.discharge_disposition_code
+left join {{ ref('terminology__admit_source') }} h
+  on c.admit_source_code = h.admit_source_code
+left join {{ ref('terminology__admit_type') }} i
+  on c.admit_type_code = i.admit_type_code
+left join {{ ref('terminology__ms_drg') }} j
+  on c.ms_drg_code = j.ms_drg_code
+left join {{ ref('terminology__apr_drg') }} k
+  on c.apr_drg_code = k.apr_drg_code
+left join {{ ref('terminology__icd_10_cm')}} icd10cm
+  on c.diagnosis_code_1 = icd10cm.icd_10_cm
+  and c.diagnosis_code_type = 'icd-10-cm'
+left join {{ ref('terminology__icd_9_cm')}} icd9cm
+  on c.diagnosis_code_1 = icd9cm.icd_9_cm
   and c.diagnosis_code_type = 'icd-9-cm'
diff --git a/models/core/staging/core__stg_claims_condition.sql b/models/core/staging/core__stg_claims_condition.sql
@@ -505,7 +505,7 @@ select distinct
         end as {{ dbt.type_string() }}
       ) as normalized_code_type
     , cast(icd.icd_10_cm as {{ dbt.type_string() }} ) as normalized_code
-    , cast(icd.description as {{ dbt.type_string() }} ) as normalized_description
+    , cast(icd.long_description as {{ dbt.type_string() }} ) as normalized_description
     , cast(unpivot_cte.diagnosis_rank as {{ dbt.type_int() }} ) as condition_rank
     , cast(unpivot_cte.present_on_admit_code as {{ dbt.type_string() }} ) as present_on_admit_code
     , cast(poa.present_on_admit_description as {{ dbt.type_string() }} ) as present_on_admit_description

diff --git a/seeds/terminology/terminology__admit_source.csv b/seeds/terminology/terminology__admit_source.csv
@@ -1,17 +1 @@
-admit_source_code,admit_source_description,newborn_description
-0,Anomaly,null
-1,Non-Health Care Facility Point of Origin (Physician Referral),Normal delivery
-2,Clinic Referral,Premature delivery
-3,HMO Referral,Sick baby
-4,Transfer from Hospital (different facility),Extramural birth
-5,Transfer from Skilled Nursing Facility (SNF),null
-6,Transfer from Another Health Care Facility,null
-7,Emergency Room,null
-8,Court/Law Enforcement,null
-9,Information Not Available,null
-A,Reserved for National Assignment,null
-B,Transfer from Another Home Health Agency,null
-C,Readmission to Same Home Health Agency,null
-D,Transfer from hospital inpatient in the same facility resulting in a separate claim to the payer,null
-E,Transfer from Ambulatory Surgical Center,null
-F,Transfer from Hospice,null
+admit_source_code,admit_source_description,newborn_description
diff --git a/seeds/terminology/terminology__admit_type.csv b/seeds/terminology/terminology__admit_type.csv
@@ -1,6 +1 @@
-admit_type_code,admit_type_description
-1,Emergency
-2,Urgent
-3,Elective
-4,Newborn
-5,Trauma Center
+admit_type_code,admit_type_description