From aad2a1e53b875a413ff2bcd8d55ca0f35d5abbd8 Mon Sep 17 00:00:00 2001 From: Vasili Zakharko Date: Mon, 22 Jul 2024 17:17:29 +0300 Subject: [PATCH 1/3] feature: create base tpdm/edfi_3 models iteration 2 --- models/staging/edfi_3/base/_edfi_3__base.yml | 15 +++++ ...se_ef3__performance_evaluation_ratings.sql | 16 +++++ .../base_ef3__performance_evaluations.sql | 16 +++++ .../staging/edfi_3/base/base_ef3__persons.sql | 16 +++++ .../base/base_ef3__rubric_dimensions.sql | 16 +++++ ...f3__survey_section_aggregate_responses.sql | 16 +++++ models/staging/tpdm/base/_tpdm__base.yml | 50 ++++++++++++---- .../staging/tpdm/base/base_tpdm__persons.sql | 16 +++++ ...base_tpdm__post_secondary_institutions.sql | 34 +++++++++++ .../staging/tpdm/base/base_tpdm__schools.sql | 52 ++++++++++++++++ .../staging/tpdm/base/base_tpdm__sections.sql | 52 ++++++++++++++++ .../base_tpdm__student_academic_records.sql | 59 +++++++++++++++++++ .../staging/tpdm/base/base_tpdm__students.sql | 43 ++++++++++++++ .../tpdm/base/base_tpdm__survey_responses.sql | 35 +++++++++++ ...dm__survey_section_aggregate_responses.sql | 16 +++++ .../base_tpdm__survey_section_responses.sql | 23 ++++++++ .../tpdm/base/base_tpdm__survey_sections.sql | 21 +++++++ 17 files changed, 486 insertions(+), 10 deletions(-) create mode 100644 models/staging/edfi_3/base/base_ef3__performance_evaluation_ratings.sql create mode 100644 models/staging/edfi_3/base/base_ef3__performance_evaluations.sql create mode 100644 models/staging/edfi_3/base/base_ef3__persons.sql create mode 100644 models/staging/edfi_3/base/base_ef3__rubric_dimensions.sql create mode 100644 models/staging/edfi_3/base/base_ef3__survey_section_aggregate_responses.sql create mode 100644 models/staging/tpdm/base/base_tpdm__persons.sql create mode 100644 models/staging/tpdm/base/base_tpdm__post_secondary_institutions.sql create mode 100644 models/staging/tpdm/base/base_tpdm__schools.sql create mode 100644 models/staging/tpdm/base/base_tpdm__sections.sql create mode 100644 models/staging/tpdm/base/base_tpdm__student_academic_records.sql create mode 100644 models/staging/tpdm/base/base_tpdm__students.sql create mode 100644 models/staging/tpdm/base/base_tpdm__survey_responses.sql create mode 100644 models/staging/tpdm/base/base_tpdm__survey_section_aggregate_responses.sql create mode 100644 models/staging/tpdm/base/base_tpdm__survey_section_responses.sql create mode 100644 models/staging/tpdm/base/base_tpdm__survey_sections.sql diff --git a/models/staging/edfi_3/base/_edfi_3__base.yml b/models/staging/edfi_3/base/_edfi_3__base.yml index 50f4ff4..859f5a3 100644 --- a/models/staging/edfi_3/base/_edfi_3__base.yml +++ b/models/staging/edfi_3/base/_edfi_3__base.yml @@ -79,9 +79,21 @@ models: - name: base_ef3__parents config: tags: ['core'] + - name: base_ef3__performance_evaluation_ratings + config: + tags: ['core ???'] + - name: base_ef3__performance_evaluations + config: + tags: ['core ???'] + - name: base_ef3__persons + config: + tags: ['core'] - name: base_ef3__programs config: tags: ['core'] + - name: base_ef3__rubric_dimensions + config: + tags: ['core'] - name: base_ef3__schools config: tags: ['core'] @@ -194,6 +206,9 @@ models: - name: base_ef3__survey_sections config: tags: ['survey'] + - name: base_ef3__survey_section_aggregate_responses + config: + tags: ['survey'] - name: base_ef3__survey_section_responses config: tags: ['survey'] diff --git a/models/staging/edfi_3/base/base_ef3__performance_evaluation_ratings.sql b/models/staging/edfi_3/base/base_ef3__performance_evaluation_ratings.sql new file mode 100644 index 0000000..2e75141 --- /dev/null +++ b/models/staging/edfi_3/base/base_ef3__performance_evaluation_ratings.sql @@ -0,0 +1,16 @@ +with performance_evaluation_ratings as ( + {{ source_edfi3('performance_evaluation_ratings') }} +), +renamed as ( + select + tenant_code, + api_year, + pull_timestamp, + last_modified_timestamp, + filename, + file_row_number, + is_deleted, + ?? + from performance_evaluation_ratings +) +select * from renamed diff --git a/models/staging/edfi_3/base/base_ef3__performance_evaluations.sql b/models/staging/edfi_3/base/base_ef3__performance_evaluations.sql new file mode 100644 index 0000000..ce3251d --- /dev/null +++ b/models/staging/edfi_3/base/base_ef3__performance_evaluations.sql @@ -0,0 +1,16 @@ +with performance_evaluations as ( + {{ source_edfi3('performance_evaluations') }} +), +renamed as ( + select + tenant_code, + api_year, + pull_timestamp, + last_modified_timestamp, + filename, + file_row_number, + is_deleted, + ?? + from performance_evaluations +) +select * from renamed diff --git a/models/staging/edfi_3/base/base_ef3__persons.sql b/models/staging/edfi_3/base/base_ef3__persons.sql new file mode 100644 index 0000000..4937abc --- /dev/null +++ b/models/staging/edfi_3/base/base_ef3__persons.sql @@ -0,0 +1,16 @@ +with persons as ( + {{ source_edfi3('persons') }} +), +renamed as ( + select + tenant_code, + api_year, + pull_timestamp, + last_modified_timestamp, + filename, + file_row_number, + is_deleted, + ??? + from persons +) +select * from renamed diff --git a/models/staging/edfi_3/base/base_ef3__rubric_dimensions.sql b/models/staging/edfi_3/base/base_ef3__rubric_dimensions.sql new file mode 100644 index 0000000..e39905c --- /dev/null +++ b/models/staging/edfi_3/base/base_ef3__rubric_dimensions.sql @@ -0,0 +1,16 @@ +with rubric_dimensions as ( + {{ source_edfi3('rubric_dimensions') }} +), +renamed as ( + select + tenant_code, + api_year, + pull_timestamp, + last_modified_timestamp, + filename, + file_row_number, + is_deleted, + ??? + from rubric_dimensions +) +select * from renamed diff --git a/models/staging/edfi_3/base/base_ef3__survey_section_aggregate_responses.sql b/models/staging/edfi_3/base/base_ef3__survey_section_aggregate_responses.sql new file mode 100644 index 0000000..c48bea1 --- /dev/null +++ b/models/staging/edfi_3/base/base_ef3__survey_section_aggregate_responses.sql @@ -0,0 +1,16 @@ +with survey_section_aggregate_responses as ( + {{ source_edfi3('survey_section_aggregate_responses') }} +), +renamed as ( + select + tenant_code, + api_year, + pull_timestamp, + last_modified_timestamp, + file_row_number, + filename, + is_deleted, + ?? + from survey_section_aggregate_responses +) +select * from renamed diff --git a/models/staging/tpdm/base/_tpdm__base.yml b/models/staging/tpdm/base/_tpdm__base.yml index 684b62e..e603a7e 100644 --- a/models/staging/tpdm/base/_tpdm__base.yml +++ b/models/staging/tpdm/base/_tpdm__base.yml @@ -7,39 +7,69 @@ models: - name: base_tpdm__educator_preparation_programs config: tags: ['tpdm'] - - name: base_tpdm__survey_response_person_target_associations - config: - tags: ['tpdm'] - name: base_tpdm__candidate_educator_preparation_program_associations config: tags: ['tpdm'] - name: base_tpdm__financial_aids config: tags: ['tpdm'] - - name: base_tpdm__rubric_dimensions + - name: base_tpdm__evaluations config: tags: ['tpdm'] - - name: base_tpdm__evaluations + - name: base_tpdm__evaluation_elements + config: + tags: ['tpdm'] + - name: base_tpdm__evaluation_element_ratings config: tags: ['tpdm'] - name: base_tpdm__evaluation_ratings config: tags: ['tpdm'] - - name: base_tpdm__performance_evaluations + - name: base_tpdm__evaluation_objectives + config: + tags: ['tpdm'] + - name: base_tpdm__evaluation_objective_ratings config: tags: ['tpdm'] - name: base_tpdm__performance_evaluation_ratings config: tags: ['tpdm'] - - name: base_tpdm__evaluation_objectives + - name: base_tpdm__performance_evaluations config: tags: ['tpdm'] - - name: base_tpdm__evaluation_objective_ratings + - name: base_tpdm__persons config: tags: ['tpdm'] - - name: base_tpdm__evaluation_elements + - name: base_tpdm__post_secondary_institutions config: tags: ['tpdm'] - - name: base_tpdm__evaluation_element_ratings + - name: base_tpdm__rubric_dimensions + config: + tags: ['tpdm'] + - name: base_tpdm__schools + config: + tags: ['tpdm'] + - name: base_tpdm__sections + config: + tags: ['tpdm'] + - name: base_tpdm__student_academic_records + config: + tags: ['tpdm'] + - name: base_tpdm__students + config: + tags: ['tpdm'] + - name: base_tpdm__survey_response_person_target_associations + config: + tags: ['tpdm'] + - name: base_tpdm__survey_responses + config: + tags: ['tpdm'] + - name: base_tpdm__survey_section_aggregate_responses + config: + tags: ['tpdm'] + - name: base_tpdm__survey_section_responses + config: + tags: ['tpdm'] + - name: base_tpdm__survey_sections config: tags: ['tpdm'] diff --git a/models/staging/tpdm/base/base_tpdm__persons.sql b/models/staging/tpdm/base/base_tpdm__persons.sql new file mode 100644 index 0000000..4937abc --- /dev/null +++ b/models/staging/tpdm/base/base_tpdm__persons.sql @@ -0,0 +1,16 @@ +with persons as ( + {{ source_edfi3('persons') }} +), +renamed as ( + select + tenant_code, + api_year, + pull_timestamp, + last_modified_timestamp, + filename, + file_row_number, + is_deleted, + ??? + from persons +) +select * from renamed diff --git a/models/staging/tpdm/base/base_tpdm__post_secondary_institutions.sql b/models/staging/tpdm/base/base_tpdm__post_secondary_institutions.sql new file mode 100644 index 0000000..9ff872b --- /dev/null +++ b/models/staging/tpdm/base/base_tpdm__post_secondary_institutions.sql @@ -0,0 +1,34 @@ +with post_secondary_institutions as ( + {{ source_edfi3('post_secondary_institutions') }} +), +renamed as ( + select + tenant_code, + api_year, + pull_timestamp, + last_modified_timestamp, + file_row_number, + filename, + is_deleted, + + v:id::string as record_guid, + v:postSecondaryInstitutionId::int as post_secondary_institution_id, + v:nameOfInstitution::string as name_of_institution, + v:shortNameOfInstitution::string as short_name_of_institution, + v:webSite::string as web_site, + -- descriptors + {{ extract_descriptor('v:postSecondaryInstitutionLevelDescriptor::string')}} as post_secondary_institution_level, + {{ extract_descriptor('v:administrativeFundingControlDescriptor::string')}} as administrative_funding_control, + {{ extract_descriptor('v:federalLocaleCodeDescriptor::string')}} as federal_locale_code, + {{ extract_descriptor('v:operationalStatusDescriptor::string')}} as operational_status_descriptor, + -- unflattened lists + v:categories as v_categories, + v:addresses as v_addresses, + v:identificationCodes as v_identification_codes, + v:indicators as v_indicators, + v:institutionTelephones as v_institution_telephones, + v:internationalAddresses as v_international_addresses, + v:mediumOfInstructions as v_medium_of_instructions + from post_secondary_institutions +) +select * from renamed diff --git a/models/staging/tpdm/base/base_tpdm__schools.sql b/models/staging/tpdm/base/base_tpdm__schools.sql new file mode 100644 index 0000000..a1ba275 --- /dev/null +++ b/models/staging/tpdm/base/base_tpdm__schools.sql @@ -0,0 +1,52 @@ +with schools as ( + {{ source_edfi3('schools') }} +), +renamed as ( + select + tenant_code, + api_year, + pull_timestamp, + last_modified_timestamp, + file_row_number, + filename, + is_deleted, + v:id::string as record_guid, + v:schoolId::integer as school_id, + v:nameOfInstitution::string as school_name, + v:shortNameOfInstitution::string as school_short_name, + v:webSite::string as website, + v:localEducationAgencyReference:localEducationAgencyId as lea_id, + -- pull out school categories + case + when array_size(v:schoolCategories) = 1 + then {{ extract_descriptor('v:schoolCategories[0]:schoolCategoryDescriptor::string') }} + when array_size(v:schoolCategories) > 1 + then 'Multiple Categories' + else NULL + end as school_category, + -- descriptors + {{ extract_descriptor('v:schoolTypeDescriptor::string') }} as school_type, + {{ extract_descriptor('v:operationalStatusDescriptor::string') }} as operational_status, + {{ extract_descriptor('v:administrativeFundingControlDescriptor::string') }} as administrative_funding_control, + {{ extract_descriptor('v:internetAccessDescriptor::string') }} as internet_access, + {{ extract_descriptor('v:titleIPartASchoolDesignationDescriptor::string') }} as title_i_part_a_school_designation, + {{ extract_descriptor('v:charterStatusDescriptor::string') }} as charter_status, + {{ extract_descriptor('v:charterApprovalAgencyTypeDescriptor::string') }} as charter_approval_agency, + {{ extract_descriptor('v:magnetSpecialProgramEmphasisSchoolDescriptor::string') }} as magnet_type, + -- references + v:localEducationAgencyReference as local_education_agency_reference, + -- unflattened lists + v:addresses as v_addresses, + v:educationOrganizationCategories as v_education_organization_categories, + v:gradeLevels as v_grade_levels, + v:identificationCodes as v_identification_codes, + v:indicators as v_indicators, + v:institutionTelephones as v_institution_telephones, + v:internationalAddresses as v_international_addresses, + v:schoolCategories as v_school_categories, + + -- edfi extensions + v:_ext as v_ext + from schools +) +select * from renamed diff --git a/models/staging/tpdm/base/base_tpdm__sections.sql b/models/staging/tpdm/base/base_tpdm__sections.sql new file mode 100644 index 0000000..7483977 --- /dev/null +++ b/models/staging/tpdm/base/base_tpdm__sections.sql @@ -0,0 +1,52 @@ +with sections as ( + {{ source_edfi3('sections') }} +), +renamed as ( + select + tenant_code, + api_year, + pull_timestamp, + last_modified_timestamp, + file_row_number, + filename, + is_deleted, + v:id::string as record_guid, + v:sectionIdentifier::string as section_id, + v:sectionName::string as section_name, + -- course offering key + v:courseOfferingReference:schoolId::integer as school_id, + v:courseOfferingReference:localCourseCode::string as local_course_code, + v:courseOfferingReference:sessionName::string as session_name, + v:courseOfferingReference:schoolYear::integer as school_year, + -- location key + v:locationReference:classroomIdentificationCode::string as classroom_identification_code, + v:locationReference:schoolId::integer as classroom_location_school_id, + -- location school key + v:locationSchoolReference:schoolId::integer as school_location_school_id, + -- values + v:availableCreditConversion::float as available_credit_conversion, + v:availableCredits::float as available_credits, + v:sequenceOfCourse::integer as sequence_of_course, + v:officialAttendancePeriod::boolean as is_official_attendance_period, + -- descriptors + {{ extract_descriptor('v:availableCreditTypeDescriptor::string') }} as available_credit_type, + {{ extract_descriptor('v:educationalEnvironmentDescriptor::string') }} as educational_environment_type, + {{ extract_descriptor('v:instructionLanguageDescriptor::string') }} as instruction_language, + {{ extract_descriptor('v:mediumOfInstructionDescriptor::string') }} as medium_of_instruction, + {{ extract_descriptor('v:populationServedDescriptor::string') }} as population_served, + -- references + v:courseOfferingReference as course_offering_reference, + v:locationReference as location_reference, + v:locationSchoolReference as location_school_reference, + -- lists + v:characteristics as v_section_characteristics, + v:classPeriods as v_class_periods, + v:courseLevelCharacteristics as v_course_level_characteristics, + v:offeredGradeLevels as v_offered_grade_levels, + v:programs as v_programs, + + -- edfi extensions + v:_ext as v_ext + from sections +) +select * from renamed diff --git a/models/staging/tpdm/base/base_tpdm__student_academic_records.sql b/models/staging/tpdm/base/base_tpdm__student_academic_records.sql new file mode 100644 index 0000000..f83df83 --- /dev/null +++ b/models/staging/tpdm/base/base_tpdm__student_academic_records.sql @@ -0,0 +1,59 @@ +with academic_records as ( + {{ source_edfi3('student_academic_records') }} +), +renamed as ( + select + tenant_code, + api_year, + pull_timestamp, + last_modified_timestamp, + file_row_number, + filename, + is_deleted, + v:id::string as record_guid, + -- identity components + v:educationOrganizationReference:educationOrganizationId::int as ed_org_id, + v:educationOrganizationReference:link:rel::string as ed_org_type, + v:schoolYearTypeReference:schoolYear::int as school_year, + v:studentReference:studentUniqueId::string as student_unique_id, + {{ extract_descriptor('v:termDescriptor::string') }} as academic_term, + -- non-identity components + v:cumulativeAttemptedCreditConversion::float as cumulative_attempted_credit_conversion, + v:cumulativeAttemptedCredits::float as cumulative_attempted_credits, + v:cumulativeEarnedCreditConversion::float as cumulative_earned_credit_conversion, + v:cumulativeEarnedCredits::float as cumulative_earned_credits, + v:projectedGraduationDate::date as projected_graduation_date, + v:sessionAttemptedCreditConversion::float as session_attempted_credit_conversion, + v:sessionAttemptedCredits::float as session_attempted_credits, + v:sessionEarnedCreditConversion::float as session_earned_credit_conversion, + v:sessionEarnedCredits::float as session_earned_credits, + v:classRanking:classRank::float as class_rank, + v:classRanking:totalNumberInClass::float as class_rank_total_students, + v:classRanking:percentageRanking::float as class_percent_rank, + v:classRanking:classRankingDate::date as class_rank_date, + -- deprecated components (use gpa list instead) + v:cumulativeGradePointAverage::float as cumulative_gpa, + v:cumulativeGradePointsEarned::float as cumulative_grade_points_earned, + v:gradeValueQualifier::string as grade_value_qualifier, + v:sessionGradePointAverage::float as session_gpa, + v:sessionGradePointsEarned::float as session_grade_points_earned, + -- descriptors + {{ extract_descriptor('v:sessionEarnedCreditTypeDescriptor::string') }} as session_earned_credit_type, + {{ extract_descriptor('v:sessionAttemptedCreditTypeDescriptor::string') }} as session_attempted_credit_type, + {{ extract_descriptor('v:cumulativeEarnedCreditTypeDescriptor::string') }} as cumulative_earned_credit_type, + {{ extract_descriptor('v:cumulativeAttemptedCreditTypeDescriptor::string') }} as cumulative_attempted_credit_type, + -- references + v:studentReference as student_reference, + v:educationOrganizationReference as education_organization_reference, + -- lists + v:academicHonors as v_academic_honors, + v:diplomas as v_diplomas, + v:gradePointAverages as v_grade_point_averages, + v:recognitions as v_recognitions, + v:reportCards as v_report_cards, + + -- edfi extensions + v:_ext as v_ext + from academic_records +) +select * from renamed diff --git a/models/staging/tpdm/base/base_tpdm__students.sql b/models/staging/tpdm/base/base_tpdm__students.sql new file mode 100644 index 0000000..ad32299 --- /dev/null +++ b/models/staging/tpdm/base/base_tpdm__students.sql @@ -0,0 +1,43 @@ +with students as ( + {{ source_edfi3('students') }} +), +renamed as ( + select + tenant_code, + api_year, + pull_timestamp, + last_modified_timestamp, + file_row_number, + filename, + is_deleted, + v:id::string as record_guid, + v:studentUniqueId::string as student_unique_id, + v:firstName::string as first_name, + v:middleName::string as middle_name, + v:lastSurname::string as last_name, + v:generationCodeSuffix::string as generation_code_suffix, + v:maidenName::string as maiden_name, + v:personalTitlePrefix::string as personal_title_prefix, + v:personReference:personId::string as person_id, + v:birthDate::date as birth_date, + v:birthCity::string as birth_city, + v:birthInternationalProvince::string as birth_international_province, + v:dateEnteredUS::date as date_entered_us, + v:multipleBirthStatus::boolean as is_multiple_birth, + -- descriptors + {{ extract_descriptor('v:personReference:sourceSystemDescriptor::string') }} as person_source_system, + {{ extract_descriptor('v:birthSexDescriptor::string') }} as birth_sex, + {{ extract_descriptor('v:citizenshipStatusDescriptor::string') }} as citizenship_status, + {{ extract_descriptor('v:birthStateAbbreviationDescriptor::string') }} as birth_state, + {{ extract_descriptor('v:birthCountryDescriptor::string') }} as birth_country, + -- nested lists + v:identificationDocuments as v_identification_documents, + v:otherNames as v_other_names, + v:personalIdentificationDocuments as v_personal_identification_documents, + v:visas as v_visas, + + -- edfi extensions + v:_ext as v_ext + from students +) +select * from renamed diff --git a/models/staging/tpdm/base/base_tpdm__survey_responses.sql b/models/staging/tpdm/base/base_tpdm__survey_responses.sql new file mode 100644 index 0000000..1d250ba --- /dev/null +++ b/models/staging/tpdm/base/base_tpdm__survey_responses.sql @@ -0,0 +1,35 @@ +with survey_responses as ( + {{ source_edfi3('survey_responses') }} +), +renamed as ( + select + tenant_code, + api_year, + pull_timestamp, + last_modified_timestamp, + file_row_number, + filename, + is_deleted, + + v:id::string as record_guid, + v:surveyReference:surveyIdentifier::string as survey_id, + v:surveyReference:namespace::string as namespace, + v:surveyResponseIdentifier::string as survey_response_id, + v:studentReference:studentUniqueId::string as student_unique_id, + v:electronicMailAddress::string as electronic_mail_address, + v:fullName::string as full_name, + v:location::string as location, + v:responseDate::date as response_date, + v:responseTime::int as completion_time_seconds, + --references + v:surveyReference as survey_reference, + v:studentReference as student_reference, + v:staffReference as staff_reference, + v:parentReference as parent_reference, + -- lists + v:surveyLevels as v_survey_levels, + -- edfi extensions + v:_ext as v_ext + from survey_responses +) +select * from renamed diff --git a/models/staging/tpdm/base/base_tpdm__survey_section_aggregate_responses.sql b/models/staging/tpdm/base/base_tpdm__survey_section_aggregate_responses.sql new file mode 100644 index 0000000..c48bea1 --- /dev/null +++ b/models/staging/tpdm/base/base_tpdm__survey_section_aggregate_responses.sql @@ -0,0 +1,16 @@ +with survey_section_aggregate_responses as ( + {{ source_edfi3('survey_section_aggregate_responses') }} +), +renamed as ( + select + tenant_code, + api_year, + pull_timestamp, + last_modified_timestamp, + file_row_number, + filename, + is_deleted, + ?? + from survey_section_aggregate_responses +) +select * from renamed diff --git a/models/staging/tpdm/base/base_tpdm__survey_section_responses.sql b/models/staging/tpdm/base/base_tpdm__survey_section_responses.sql new file mode 100644 index 0000000..78bb06f --- /dev/null +++ b/models/staging/tpdm/base/base_tpdm__survey_section_responses.sql @@ -0,0 +1,23 @@ +with section_responses as ( + {{ source_edfi3('section_responses') }} +), +renamed as ( + select + tenant_code, + api_year, + pull_timestamp, + last_modified_timestamp, + file_row_number, + filename, + is_deleted, + + v:id::string as record_guid, + v:surveyResponseReference:surveyResponseIdentifier::string as survey_response_id, + v:surveySectionReference:surveyIdentifier::string as survey_id, + v:sectionRating::decimal(9,3) as section_rating, + -- references + v:surveyResponseReference as survey_response_reference, + v:surveySectionReference as survey_section_reference + from section_responses +) +select * from renamed diff --git a/models/staging/tpdm/base/base_tpdm__survey_sections.sql b/models/staging/tpdm/base/base_tpdm__survey_sections.sql new file mode 100644 index 0000000..9b40ea5 --- /dev/null +++ b/models/staging/tpdm/base/base_tpdm__survey_sections.sql @@ -0,0 +1,21 @@ +with survey_sections as ( + {{ source_edfi3('survey_sections') }} +), +renamed as ( + select + tenant_code, + api_year, + pull_timestamp, + last_modified_timestamp, + file_row_number, + filename, + is_deleted, + + v:id::string as record_guid, + v:surveyReference::surveyIdentifier::string as survey_id, + v:surveySectionTitle::string as survey_section_title, + -- references + v:surveyReference as survey_reference + from survey_sections +) +select * from renamed From e849ae3b2a003b63818827454047db56deffd8a0 Mon Sep 17 00:00:00 2001 From: Vasili Zakharko Date: Mon, 22 Jul 2024 21:05:41 +0300 Subject: [PATCH 2/3] feature: create base tpdm/edfi_3 models iteration 2 --- .../staging/edfi_3/base/base_ef3__persons.sql | 16 ----- .../base/base_ef3__school_year_types.sql | 21 ++++++ ...e_ef3__section_attendance_taken_events.sql | 37 ++++++++++ .../base/base_ef3__staff_absence_events.sql | 26 +++++++ .../base_ef3__staff_cohort_associations.sql | 27 +++++++ ...staff_discipline_incident_associations.sql | 25 +++++++ ...tion_organization_contact_associations.sql | 71 +++++++++++++++++++ .../edfi_3/base/base_ef3__staff_leaves.sql | 27 +++++++ .../base_ef3__staff_program_associations.sql | 28 ++++++++ ...ase_ef3__student_competency_objectives.sql | 38 ++++++++++ ..._ef3__student_cte_program_associations.sql | 47 ++++++++++++ ...cipline_incident_behavior_associations.sql | 4 +- ...ine_incident_non_offender_associations.sql | 2 +- .../base_ef3__student_gradebook_entries.sql | 54 ++++++++++++++ ...ef3__student_intervention_associations.sql | 46 ++++++++++++ ...student_intervention_attendance_events.sql | 0 ...migrant_education_program_associations.sql | 55 ++++++++++++++ ...ted_or_delinquent_program_associations.sql | 49 +++++++++++++ ...ef3__student_program_attendance_events.sql | 37 ++++++++++ ...hool_food_service_program_associations.sql | 43 +++++++++++ .../base_ef3__survey_course_associations.sql | 25 +++++++ .../base_ef3__survey_program_associations.sql | 26 +++++++ .../base_ef3__survey_question_responses.sql | 2 +- .../base/base_ef3__survey_questions.sql | 2 +- ...on_organization_target_associations_v2.sql | 24 +++++++ .../base_ef3__survey_section_associations.sql | 24 +++++++ ...ation_organization_target_associations.sql | 22 ++++++ ...on_response_staff_target_associations.sql} | 7 +- .../staging/tpdm/base/base_tpdm__persons.sql | 16 ----- ...cator_preparation_program_associations.sql | 36 ++++++++++ .../tpdm/base/base_tpdm__survey_responses.sql | 2 +- ...on_response_person_target_associations.sql | 23 ++++++ .../base_tpdm__survey_section_responses.sql | 23 ------ .../tpdm/base/base_tpdm__survey_sections.sql | 21 ------ 34 files changed, 823 insertions(+), 83 deletions(-) delete mode 100644 models/staging/edfi_3/base/base_ef3__persons.sql create mode 100644 models/staging/edfi_3/base/base_ef3__school_year_types.sql create mode 100644 models/staging/edfi_3/base/base_ef3__section_attendance_taken_events.sql create mode 100644 models/staging/edfi_3/base/base_ef3__staff_absence_events.sql create mode 100644 models/staging/edfi_3/base/base_ef3__staff_cohort_associations.sql create mode 100644 models/staging/edfi_3/base/base_ef3__staff_discipline_incident_associations.sql create mode 100644 models/staging/edfi_3/base/base_ef3__staff_education_organization_contact_associations.sql create mode 100644 models/staging/edfi_3/base/base_ef3__staff_leaves.sql create mode 100644 models/staging/edfi_3/base/base_ef3__staff_program_associations.sql create mode 100644 models/staging/edfi_3/base/base_ef3__student_competency_objectives.sql create mode 100644 models/staging/edfi_3/base/base_ef3__student_cte_program_associations.sql create mode 100644 models/staging/edfi_3/base/base_ef3__student_gradebook_entries.sql create mode 100644 models/staging/edfi_3/base/base_ef3__student_intervention_associations.sql create mode 100644 models/staging/edfi_3/base/base_ef3__student_intervention_attendance_events.sql create mode 100644 models/staging/edfi_3/base/base_ef3__student_migrant_education_program_associations.sql create mode 100644 models/staging/edfi_3/base/base_ef3__student_neglected_or_delinquent_program_associations.sql create mode 100644 models/staging/edfi_3/base/base_ef3__student_program_attendance_events.sql create mode 100644 models/staging/edfi_3/base/base_ef3__student_school_food_service_program_associations.sql create mode 100644 models/staging/edfi_3/base/base_ef3__survey_course_associations.sql create mode 100644 models/staging/edfi_3/base/base_ef3__survey_program_associations.sql create mode 100644 models/staging/edfi_3/base/base_ef3__survey_response_education_organization_target_associations_v2.sql create mode 100644 models/staging/edfi_3/base/base_ef3__survey_section_associations.sql create mode 100644 models/staging/edfi_3/base/base_ef3__survey_section_response_education_organization_target_associations.sql rename models/staging/{tpdm/base/base_tpdm__survey_section_aggregate_responses.sql => edfi_3/base/base_ef3__survey_section_response_staff_target_associations.sql} (53%) delete mode 100644 models/staging/tpdm/base/base_tpdm__persons.sql create mode 100644 models/staging/tpdm/base/base_tpdm__staff_educator_preparation_program_associations.sql create mode 100644 models/staging/tpdm/base/base_tpdm__survey_section_response_person_target_associations.sql delete mode 100644 models/staging/tpdm/base/base_tpdm__survey_section_responses.sql delete mode 100644 models/staging/tpdm/base/base_tpdm__survey_sections.sql diff --git a/models/staging/edfi_3/base/base_ef3__persons.sql b/models/staging/edfi_3/base/base_ef3__persons.sql deleted file mode 100644 index 4937abc..0000000 --- a/models/staging/edfi_3/base/base_ef3__persons.sql +++ /dev/null @@ -1,16 +0,0 @@ -with persons as ( - {{ source_edfi3('persons') }} -), -renamed as ( - select - tenant_code, - api_year, - pull_timestamp, - last_modified_timestamp, - filename, - file_row_number, - is_deleted, - ??? - from persons -) -select * from renamed diff --git a/models/staging/edfi_3/base/base_ef3__school_year_types.sql b/models/staging/edfi_3/base/base_ef3__school_year_types.sql new file mode 100644 index 0000000..16c8fd4 --- /dev/null +++ b/models/staging/edfi_3/base/base_ef3__school_year_types.sql @@ -0,0 +1,21 @@ +with school_year_types as ( + {{ source_edfi3('school_year_types') }} +), +renamed as ( + select + tenant_code, + api_year, + pull_timestamp, + last_modified_timestamp, + filename, + file_row_number, + is_deleted, + + v:id::string as record_guid, + v:schoolYear::string as school_year, + v:currentSchoolYear::string as current_school_year, + v:schoolYearDescription::string as school_year_description + + from school_year_types +) +select * from renamed diff --git a/models/staging/edfi_3/base/base_ef3__section_attendance_taken_events.sql b/models/staging/edfi_3/base/base_ef3__section_attendance_taken_events.sql new file mode 100644 index 0000000..6948cb2 --- /dev/null +++ b/models/staging/edfi_3/base/base_ef3__section_attendance_taken_events.sql @@ -0,0 +1,37 @@ +with section_attendance_taken_events as ( + {{ source_edfi3('section_attendance_taken_events') }} +), +renamed as ( + select + tenant_code, + api_year, + pull_timestamp, + last_modified_timestamp, + filename, + file_row_number, + is_deleted, + + v:id::string as record_guid, + -- fields + v:eventDate::string as event_date, + -- calendar date key + v:calendarDateReference:calendarCode::string as calendar_date_local_course_code, + v:calendarDateReference:date::date as calendar_date_date, + v:calendarDateReference:schoolId::int as calendar_date_school_id, + v:calendarDateReference:schoolYear::int as calendar_date_school_year, + -- section key + v:sectionReference:localCourseCode::string as local_course_code, + v:sectionReference:schoolId::int as school_id, + v:sectionReference:schoolYear::int as school_year, + v:sectionReference:sectionIdentifier::string as section_id, + v:sectionReference:sessionName::string as session_name, + -- staff key + v:staffReference:staffUniqueId::string as staff_unique_id, + -- references + v:calendarDateReference as calendar_date_reference, + v:sectionReference as section_reference, + v:staffReference as staff_reference + + from section_attendance_taken_events +) +select * from renamed diff --git a/models/staging/edfi_3/base/base_ef3__staff_absence_events.sql b/models/staging/edfi_3/base/base_ef3__staff_absence_events.sql new file mode 100644 index 0000000..a087dba --- /dev/null +++ b/models/staging/edfi_3/base/base_ef3__staff_absence_events.sql @@ -0,0 +1,26 @@ +with staff_absence_events as ( + {{ source_edfi3('staff_absence_events') }} +), +renamed as ( + select + tenant_code, + api_year, + pull_timestamp, + last_modified_timestamp, + file_row_number, + filename, + is_deleted, + + v:id::string as record_guid, + v:eventDate::date as event_date, + v:absenceEventReason::string as absence_event_reason, + v:hoursAbsent::integer as hours_absent, + v:staffReference:staffUniqueId::integer as staff_unique_id, + -- descriptors + {{ extract_descriptor('v:absenceEventCategoryDescriptor::string') }} as absence_event_category, + -- references + v:staffReference as school_reference + + from staff_absence_events +) +select * from renamed diff --git a/models/staging/edfi_3/base/base_ef3__staff_cohort_associations.sql b/models/staging/edfi_3/base/base_ef3__staff_cohort_associations.sql new file mode 100644 index 0000000..459f1d5 --- /dev/null +++ b/models/staging/edfi_3/base/base_ef3__staff_cohort_associations.sql @@ -0,0 +1,27 @@ +with staff_cohort_associations as ( + {{ source_edfi3('staff_cohort_associations') }} +), +renamed as ( + select + tenant_code, + api_year, + pull_timestamp, + last_modified_timestamp, + file_row_number, + filename, + is_deleted, + + v:id::string as record_guid, + v:beginDate::date as begin_date, + v:endDate::date as end_date, + v:studentRecordAccess::boolean as student_record_access, + v:staffReference:staffUniqueId::integer as staff_unique_id, + v:cohortReference:cohortIdentifier::string as cohort_dentifier, + v:cohortReference:educationOrganizationId::integer as education_organization_id, + -- references + v:cohortReference as cohort_reference, + v:staffReference as staff_reference + + from staff_cohort_associations +) +select * from renamed diff --git a/models/staging/edfi_3/base/base_ef3__staff_discipline_incident_associations.sql b/models/staging/edfi_3/base/base_ef3__staff_discipline_incident_associations.sql new file mode 100644 index 0000000..1d3842e --- /dev/null +++ b/models/staging/edfi_3/base/base_ef3__staff_discipline_incident_associations.sql @@ -0,0 +1,25 @@ +with staff_discipline_incident_associations as ( + {{ source_edfi3('staff_discipline_incident_associations') }} +), +renamed as ( + select + tenant_code, + api_year, + pull_timestamp, + last_modified_timestamp, + file_row_number, + filename, + is_deleted, + + v:id::string as record_guid, + v:disciplineIncidentParticipationCodes:disciplineIncidentParticipationCodeDescriptor::string as discipline_incident_participation_code_descriptor, + v:staffReference:staffUniqueId::integer as staff_unique_id, + v:disciplineIncidentReference:incidentIdentifier::string as incident_identifier, + v:disciplineIncidentReference:schoolId::integer as school_id, + -- references + v:disciplineIncidentReference as discipline_incident_reference, + v:staffReference as staff_reference + + from staff_discipline_incident_associations +) +select * from renamed diff --git a/models/staging/edfi_3/base/base_ef3__staff_education_organization_contact_associations.sql b/models/staging/edfi_3/base/base_ef3__staff_education_organization_contact_associations.sql new file mode 100644 index 0000000..be0c972 --- /dev/null +++ b/models/staging/edfi_3/base/base_ef3__staff_education_organization_contact_associations.sql @@ -0,0 +1,71 @@ +with staff_education_organization_contact_associations as ( + {{ source_edfi3('staff_education_organization_contact_associations') }} +), +renamed as ( + select + tenant_code, + api_year, + pull_timestamp, + last_modified_timestamp, + file_row_number, + filename, + is_deleted, + + v:id::string as record_guid, + v:contactTitle::string as contact_title, + v:contactTypeDescriptor::string as contact_type_descriptor, + v:electronicMailAddress::string as electronic_mail_address, + + v:address:addressTypeDescriptor::string as address_type_descriptor, + v:address:localeDescriptor::string as address_locale_descriptor, + v:address:stateAbbreviationDescriptor::string as address_state_abbreviation_descriptor, + v:address:apartmentRoomSuiteNumber::string as address_apartment_room_suite_number, + v:address:buildingSiteNumber::string as address_building_site_number, + v:address:city::string as address_city, + v:address:congressionalDistrict::string as address_congressional_district, + v:address:countyFIPSCode::string as address_county_fips_code, + v:address:doNotPublishIndicator::boolean as is_address_do_not_publish_indicator, + v:address:latitude::string as address_latitude, + v:address:longitude::string as address_longitude, + v:address:nameOfCounty::string as address_name_of_county, + v:address:postalCode::string as address_postal_code, + v:address:streetNumberName::string as address_street_number_name, + + v:educationOrganizationReference:educationOrganizationId::integer as ed_org_id, + v:staffReference:staffUniqueId::integer as staff_unique_id, + -- lists + v:telephones as v_telephones, + v:address:periods as v_address_periods, + + -- references + v:educationOrganizationReference as educationOrganizationReference, + v:staffReference as staff_reference + + from sessions +) +select * from renamed + + + + "address": { + "addressTypeDescriptor": "string", + "localeDescriptor": "string", + "": "string", + "": "string", + "": "string", + "": "string", + "": "string", + "": "string", + "": true, + "": "string", + "": "string", + "": "string", + "": "string", + "": "string", + "periods": [ + { + "beginDate": "2024-07-23", + "endDate": "2024-07-23" + } + ] + }, diff --git a/models/staging/edfi_3/base/base_ef3__staff_leaves.sql b/models/staging/edfi_3/base/base_ef3__staff_leaves.sql new file mode 100644 index 0000000..2687691 --- /dev/null +++ b/models/staging/edfi_3/base/base_ef3__staff_leaves.sql @@ -0,0 +1,27 @@ +with staff_leaves as ( + {{ source_edfi3('staff_leaves') }} +), +renamed as ( + select + tenant_code, + api_year, + pull_timestamp, + last_modified_timestamp, + file_row_number, + filename, + is_deleted, + + v:id::string as record_guid, + v:beginDate::date as begin_date, + v:staffReference:staffUniqueId::string as staff_unique_id, + v:endDate::date as end_date, + v:reason::string as reason, + v:substituteAssigned::boolean as is_substitute_assigned, + -- descriptors + {{ extract_descriptor('v:staffLeaveEventCategoryDescriptor') }} as staff_leave_event_ñategory, + -- references + v:staffReference as staff_reference + + from staff_leaves +) +select * from renamed diff --git a/models/staging/edfi_3/base/base_ef3__staff_program_associations.sql b/models/staging/edfi_3/base/base_ef3__staff_program_associations.sql new file mode 100644 index 0000000..eecfe95 --- /dev/null +++ b/models/staging/edfi_3/base/base_ef3__staff_program_associations.sql @@ -0,0 +1,28 @@ +with staff_program_associations as ( + {{ source_edfi3('staff_program_associations') }} +), +renamed as ( + select + tenant_code, + api_year, + pull_timestamp, + last_modified_timestamp, + file_row_number, + filename, + is_deleted, + + v:id::string as record_guid, + v:beginDate::date as begin_date, + v:endDate::date as end_date, + v:staffReference:staffUniqueId::string as staff_unique_id, + v:programReference:educationOrganizationId::integer as education_organization_id, + v:programReference:programName::string as program_name, + v:programReference:programTypeDescriptor::string as program_type_descriptor, + v:studentRecordAccess::boolean as is_studen_record_access, + -- references + v:programReference as program_reference, + v:staffReference as staff_reference + + from staff_program_associations +) +select * from renamed diff --git a/models/staging/edfi_3/base/base_ef3__student_competency_objectives.sql b/models/staging/edfi_3/base/base_ef3__student_competency_objectives.sql new file mode 100644 index 0000000..2be377c --- /dev/null +++ b/models/staging/edfi_3/base/base_ef3__student_competency_objectives.sql @@ -0,0 +1,38 @@ +with student_competency_objectives as ( + {{ source_edfi3('student_competency_objectives') }} +), +renamed as ( + select + tenant_code, + api_year, + pull_timestamp, + last_modified_timestamp, + file_row_number, + filename, + is_deleted, + v:id::string as record_guid, + v:diagnosticStatement::string as diagnostic_statement, + v:studentReference:studentUniqueId::string as student_unique_id, + v:gradingPeriodReference:gradingPeriodDescriptor::integer as grading_period_descriptor, + v:gradingPeriodReference:periodSequence::integer as period_sequence, + v:gradingPeriodReference:schoolId::integer as school_id, + v:gradingPeriodReference:schoolYear::integer as school_year, + + v:objectiveCompetencyObjectiveReference:educationOrganizationId::integer as education_organization_id, + v:objectiveCompetencyObjectiveReference:objective::string as objective, + v:objectiveCompetencyObjectiveReference:objectiveGradeLevelDescriptor::string as objective_grade_level_descriptor, + -- descriptors + {{ extract_descriptor('v:competencyLevelDescriptor::string') }} as competency_level, + -- references + v:studentReference as student_reference, + v:gradingPeriodReference as grading_periodR_rference, + -- lists + v:generalStudentProgramAssociations as v_general_student_program_associations, + v:studentSectionAssociations as v_student_section_associations, + + -- edfi extensions ???? + v:_ext as v_ext + + from student_competency_objectives +) +select * from renamed diff --git a/models/staging/edfi_3/base/base_ef3__student_cte_program_associations.sql b/models/staging/edfi_3/base/base_ef3__student_cte_program_associations.sql new file mode 100644 index 0000000..bbf8c84 --- /dev/null +++ b/models/staging/edfi_3/base/base_ef3__student_cte_program_associations.sql @@ -0,0 +1,47 @@ +with student_cte_program_associations as ( + {{ source_edfi3('student_cte_program_associations') }} +), +renamed as ( + select + tenant_code, + api_year, + pull_timestamp, + last_modified_timestamp, + file_row_number, + filename, + is_deleted, + v:id::string as record_guid, + + v:beginDate::date as begin_date, + v:endDate::date as end_date, + v:nonTraditionalGenderStatus::boolean as is_non_traditional_gender_status, + v:privateCTEProgram::boolean as is_private_c_t_e_program, + v:servedOutsideOfRegularSession::boolean as served_outside_of_regular_session, + v:participationStatus:participationStatusDescriptor::string as participation_status_descriptor, + v:participationStatus:designatedBy::string as designated_by + v:participationStatus:statusBeginDate::date as status_begin_date + v:participationStatus:statusEndDate::date as status_end_date, + v:studentReference:studentUniqueId::string as student_unique_id, + v:educationOrganizationReference:educationOrganizationId::integer as education_organization_id, + v:programReference:educationOrganizationId::integer as education_organization_id + v:programReference:programName::integer as program_name + v:programReference:programTypeDescriptor::integer as program_type_descriptor + -- descriptor + {{ extract_descriptor('v:reasonExitedDescriptor::string') }} as reason_exited, + {{ extract_descriptor('v:technicalSkillsAssessmentDescriptor::string') }} as technical_skills_assessment, + -- references + v:studentReference as student_reference, + v:gradingPeriodReference as grading_period_reference, + -- lists + v:ctePrograms as v_cte_programs, + v:cteProgramServices as v_cte_program_services, + v:studentSectionAssociations as v_student_section_associations, + v:programParticipationStatuses as v_programParticipationStatuses + v:services as v_services, + + -- edfi extensions ???? + v:_ext as v_ext + + from student_cte_program_associations +) +select * from renamed diff --git a/models/staging/edfi_3/base/base_ef3__student_discipline_incident_behavior_associations.sql b/models/staging/edfi_3/base/base_ef3__student_discipline_incident_behavior_associations.sql index 862927f..10386cf 100644 --- a/models/staging/edfi_3/base/base_ef3__student_discipline_incident_behavior_associations.sql +++ b/models/staging/edfi_3/base/base_ef3__student_discipline_incident_behavior_associations.sql @@ -19,7 +19,7 @@ renamed as ( {{ extract_descriptor('v:behaviorDescriptor::string') }} as behavior_type, -- references v:disciplineIncidentReference as discipline_incident_reference, - v:studentReference as student_reference, + v:studentReference as student_reference, -- lists v:disciplineIncidentParticipationCodes as v_discipline_incident_participation_codes, @@ -27,4 +27,4 @@ renamed as ( v:_ext as v_ext from student_discipline_incident_behavior ) -select * from renamed \ No newline at end of file +select * from renamed diff --git a/models/staging/edfi_3/base/base_ef3__student_discipline_incident_non_offender_associations.sql b/models/staging/edfi_3/base/base_ef3__student_discipline_incident_non_offender_associations.sql index 168b6ee..ca9af59 100644 --- a/models/staging/edfi_3/base/base_ef3__student_discipline_incident_non_offender_associations.sql +++ b/models/staging/edfi_3/base/base_ef3__student_discipline_incident_non_offender_associations.sql @@ -24,4 +24,4 @@ renamed as ( v:_ext as v_ext from student_discipline_incident_nonoffender ) -select * from renamed \ No newline at end of file +select * from renamed diff --git a/models/staging/edfi_3/base/base_ef3__student_gradebook_entries.sql b/models/staging/edfi_3/base/base_ef3__student_gradebook_entries.sql new file mode 100644 index 0000000..793510e --- /dev/null +++ b/models/staging/edfi_3/base/base_ef3__student_gradebook_entries.sql @@ -0,0 +1,54 @@ +with student_gradebook_entries as ( + {{ source_edfi3('student_gradebook_entries') }} +), + +renamed as ( + select + -- generic columns + tenant_code, + api_year, + pull_timestamp, + last_modified_timestamp, + file_row_number, + filename, + is_deleted, + + v:id::string as record_guid, + + v:dateFulfilled::date as date_fulfilled, + v:diagnosticStatement::string as diagnostic_statement, + v:letterGradeEarned::string as letter_grade_earned, + v:numericGradeEarned::integer as numeric_grade_earned, + v:studentReference:studentUniqueId::string as student_unique_id, + + v:gradebookEntryReference:dateAssigned::date as date_assigned, + v:gradebookEntryReference:gradebookEntryTitle::string as gradebook_entry_title, + v:gradebookEntryReference:localCourseCode: string as local_course_code, + v:gradebookEntryReference:schoolId::integer as school_id, + v:gradebookEntryReference:schoolYear::integer as school_year, + v:gradebookEntryReference:sectionIdentifier::string as section_identifier, + v:gradebookEntryReference:sessionName::string as session_name, + + v:studentSectionAssociationReference:sessionName::string as session_name, + v:studentSectionAssociationReference:beginDate::date as begin_date, + v:studentSectionAssociationReference:localCourseCode::string as local_course_code, + v:studentSectionAssociationReference:schoolId::integer as school_id, + v:studentSectionAssociationReference:schoolYear::integer as school_year, + v:studentSectionAssociationReference:sectionIdentifier::string as section_identifier, + v:studentSectionAssociationReference:sessionName::string as session_name, + v:studentSectionAssociationReference:studentUniqueId::string as student_unique_id, + + -- descriptors + {{ extract_descriptor('v:competencyLevelDescriptor::string') }} as competency_level, + + -- references + v:educationOrganizationReference as education_organization_reference, + v:studentReference as student_reference, + + -- edfi extensions + v:_ext as v_ext + + from source_stu_responsibility +) + +select * from renamed diff --git a/models/staging/edfi_3/base/base_ef3__student_intervention_associations.sql b/models/staging/edfi_3/base/base_ef3__student_intervention_associations.sql new file mode 100644 index 0000000..31e1529 --- /dev/null +++ b/models/staging/edfi_3/base/base_ef3__student_intervention_associations.sql @@ -0,0 +1,46 @@ +with source_stu_programs as ( + {{ source_edfi3('student_homeless_program_associations') }} +), + +renamed as ( + select + -- generic columns + tenant_code, + api_year, + pull_timestamp, + last_modified_timestamp, + file_row_number, + filename, + is_deleted, + + v:id::string as record_guid, + v:diagnosticStatement::string as diagnostic_statement, + v:dosage::integer as dosage, + + v:studentReference:studentUniqueId::string as student_unique_id, + + v:interventionReference:educationOrganizationId::integer as education_organization_id, + v:interventionReference:interventionIdentificationCode::string as intervention_identification_code, + + v:cohortReference:cohortIdentifier::string as ed_org_id, + v:cohortReference:educationOrganizationId::integer as intervention_identification_code, + + v:participationStatus:designatedBy::string as designated_by, + v:participationStatus:statusBeginDate::date as status_begin_date, + v:participationStatus:statusEndDate::date as status_end_date, + + -- references + v:interventionReference as intervention_reference, + v:cohortReference as cohort_reference, + v:studentReference as student_reference, + + -- lists + v:interventionEffectivenesses as v_intervention_effectivenesses, + + -- edfi extensions + v:_ext as v_ext + + from source_stu_programs +) + +select * from renamed diff --git a/models/staging/edfi_3/base/base_ef3__student_intervention_attendance_events.sql b/models/staging/edfi_3/base/base_ef3__student_intervention_attendance_events.sql new file mode 100644 index 0000000..e69de29 diff --git a/models/staging/edfi_3/base/base_ef3__student_migrant_education_program_associations.sql b/models/staging/edfi_3/base/base_ef3__student_migrant_education_program_associations.sql new file mode 100644 index 0000000..b18f7dc --- /dev/null +++ b/models/staging/edfi_3/base/base_ef3__student_migrant_education_program_associations.sql @@ -0,0 +1,55 @@ +with student_neglected_or_delinquent_program_associations as ( + {{ source_edfi3('student_neglected_or_delinquent_program_associations') }} +), +renamed as ( + select + tenant_code, + api_year, + pull_timestamp, + last_modified_timestamp, + file_row_number, + filename, + is_deleted, + + v:id::string as record_guid, + v:beginDate::date as begin_date, + v:endDate::date as end_date, + + v:eligibilityExpirationDate::date as eligibility_expiration_date, + v:lastQualifyingMove::date as last_qualifying_move, + v:priorityForServices::boolean as is_priority_for_services, + v:qualifyingArrivalDate::date as qualifying_arrival_date, + v:servedOutsideOfRegularSession::boolean as is_served_outside_of_regular_session + v:stateResidencyDate::date as state_residency_date + v:usInitialEntry::date as us_initial_entry + v:usInitialSchoolEntry:date as us_initial_school_entry + v:usMostRecentEntry::date as us_most_recent_entry + + v:educationOrganizationReference:educationOrganizationId::integer as ed_org_id, + + v:studentReference:studentUniqueId::string as student_unique_id, + + v:programReference:educationOrganizationId::integer as education_organization_id, + v:programReference:programName::string as program_name, + v:programReference:programTypeDescriptor::string as program_type_descriptor, + + v:participationStatus:participationStatusDescriptor::string as participation_status_descriptor, + v:participationStatus:designatedBy::string as designated_by, + v:participationStatus:statusBeginDate::date as status_begin_date, + v:participationStatus:statusEndDate::date as status_end_date, + -- descriptors + {{ extract_descriptor('v:continuationOfServicesReasonDescriptor::string') }} as continuation_of_services_reason_descriptor, + {{ extract_descriptor('v:reasonExitedDescriptor::string') }} as reason_exited, + -- references + v:educationOrganizationReference as parent_reference, + v:programReference as programReference, + v:parentReference as parent_reference, + v:studentReference as student_reference, + -- list + v:migrantEducationProgramServices as v_migrantEducationProgramServices, + + -- edfi extensions + v:_ext as v_ext + from student_neglected_or_delinquent_program_associations +) +select * from renamed diff --git a/models/staging/edfi_3/base/base_ef3__student_neglected_or_delinquent_program_associations.sql b/models/staging/edfi_3/base/base_ef3__student_neglected_or_delinquent_program_associations.sql new file mode 100644 index 0000000..62d55be --- /dev/null +++ b/models/staging/edfi_3/base/base_ef3__student_neglected_or_delinquent_program_associations.sql @@ -0,0 +1,49 @@ +with student_neglected_or_delinquent_program_associations as ( + {{ source_edfi3('student_neglected_or_delinquent_program_associations') }} +), +renamed as ( + select + tenant_code, + api_year, + pull_timestamp, + last_modified_timestamp, + file_row_number, + filename, + is_deleted, + + v:id::string as record_guid, + v:beginDate::date as begin_date, + v:endDate::date as end_date, + v:servedOutsideOfRegularSession::boolean as is_served_outside_of_regular_session, + + v:educationOrganizationReference:educationOrganizationId::integer as ed_org_id, + + v:studentReference:studentUniqueId::string as student_unique_id, + + v:programReference:educationOrganizationId::integer as education_organization_id, + v:programReference:programName::string as program_name, + v:programReference:programTypeDescriptor::string as program_type_descriptor, + + v:participationStatus:participationStatusDescriptor::string as participation_status_descriptor, + v:participationStatus:designatedBy::string as designated_by, + v:participationStatus:statusBeginDate::date as status_begin_date, + v:participationStatus:statusEndDate::date as status_end_date, + -- descriptors + {{ extract_descriptor('v:elaProgressLevelDescriptor::string') }} as mathematics_progress_level, + {{ extract_descriptor('v:mathematicsProgressLevelDescriptor::string') }} as mathematics_progress_level, + {{ extract_descriptor('v:neglectedOrDelinquentProgramDescriptor::string') }} as neglected_or_delinquent_program, + {{ extract_descriptor('v:reasonExitedDescriptor::string') }} as reason_exited, + -- references + v:educationOrganizationReference as parent_reference, + v:programReference as programReference, + v:parentReference as parent_reference, + v:studentReference as student_reference, + -- list + v:neglectedOrDelinquentProgramServices as v_neglected_or_delinquent_program_services, + v:programParticipationStatuses as v_program_participation_statuses, + + -- edfi extensions + v:_ext as v_ext + from student_neglected_or_delinquent_program_associations +) +select * from renamed diff --git a/models/staging/edfi_3/base/base_ef3__student_program_attendance_events.sql b/models/staging/edfi_3/base/base_ef3__student_program_attendance_events.sql new file mode 100644 index 0000000..bf26505 --- /dev/null +++ b/models/staging/edfi_3/base/base_ef3__student_program_attendance_events.sql @@ -0,0 +1,37 @@ +with student_program_attendance_events as ( + {{ source_edfi3('student_program_attendance_events') }} +), +renamed as ( + select + -- generic columns + tenant_code, + api_year, + pull_timestamp, + last_modified_timestamp, + file_row_number, + filename, + is_deleted, + + v:id::string as record_guid, + v:eventDate::date as event_date + v:educationOrganizationReference:educationOrganizationId::integer as ed_org_id, + v:educationOrganizationReference:link:rel::string as ed_org_type, + v:programReference:educationOrganizationId::integer as program_ed_org_id, + v:programReference:programName::string as program_name, + v:studentReference:studentUniqueId::string as student_id, + v:attendanceEventReason::string as attendance_event_reason, + v:eventDuration::integer as event_duration, + v:programAttendanceDuration::integer as program_attendance_duration, + + -- descriptors + {{ extract_descriptor('v:attendanceEventCategoryDescriptor::string') }} as attendance_event_category, + {{ extract_descriptor('v:educationalEnvironmentDescriptor::string') }} as educational_environment, + + -- references + v:educationOrganizationReference as education_organization_reference, + v:programReference as program_reference, + v:studentReference as student_reference + + from source_stu_programs +) +select * from renamed diff --git a/models/staging/edfi_3/base/base_ef3__student_school_food_service_program_associations.sql b/models/staging/edfi_3/base/base_ef3__student_school_food_service_program_associations.sql new file mode 100644 index 0000000..d5957fc --- /dev/null +++ b/models/staging/edfi_3/base/base_ef3__student_school_food_service_program_associations.sql @@ -0,0 +1,43 @@ +with student_school_food_service_program_associations as ( + {{ source_edfi3('student_school_food_service_program_associations') }} +), +renamed as ( + select + tenant_code, + api_year, + pull_timestamp, + last_modified_timestamp, + filename, + file_row_number, + is_deleted, + + v:id::string as record_guid, + v:beginDate::date as begin_date, + v:endDate::date as end_date, + v:directCertification::boolean as is_direct_certification, + v:servedOutsideOfRegularSession::boolean as is_served_outside_of_regular_session, + + v:educationOrganizationReference:education_organization_id::integer as education_organization_id, + v:programReference:educationOrganizationId:integer as education_organization_id, + v:programReference:programName::string as program_name, + v:programReference:programTypeDescriptor::string as program_type_descriptor, + v:studentReference:studentUniqueId::string as student_unique_id, + + v:participationStatus:participationStatusDescriptor::string as participation_status_descriptor, + v:participationStatus:designatedBy::string as designated_by, + v:participationStatus:statusBeginDate::date as status_begin_date, + v:participationStatus:statusEndDate::date as status_end_date, + + -- descriptors + {{ extract_descriptor('v:reasonExitedDescriptor::string') }} as reason_exited_descriptor, + -- references + v:educationOrganizationReference as education_organization_reference, + v:programReference as program_reference, + v:studentReference as student_reference + -- nested lists + v:programParticipationStatuses as v_programParticipationStatuses, + v:schoolFoodServiceProgramServices as v_schoolFoodServiceProgramServices + + from student_school_food_service_program_associations +) +select * from renamed diff --git a/models/staging/edfi_3/base/base_ef3__survey_course_associations.sql b/models/staging/edfi_3/base/base_ef3__survey_course_associations.sql new file mode 100644 index 0000000..2466511 --- /dev/null +++ b/models/staging/edfi_3/base/base_ef3__survey_course_associations.sql @@ -0,0 +1,25 @@ +with survey_question_responses as ( + {{ source_edfi3('survey_question_responses') }} +), +renamed as ( + select + tenant_code, + api_year, + pull_timestamp, + last_modified_timestamp, + file_row_number, + filename, + is_deleted, + + v:id::string as record_guid, + v:courseReference:courseCode::string as course_code, + v:courseReference:educationOrganizationId::integer as education_organization_id, + v:surveyReference:namespace::string as namespace, + v:surveyReference:surveyIdentifier::string as survey_identifier, + -- references + v:courseReference as program_reference, + v:surveyReference as survey_reference + + from survey_question_responses +) +select * from renamed diff --git a/models/staging/edfi_3/base/base_ef3__survey_program_associations.sql b/models/staging/edfi_3/base/base_ef3__survey_program_associations.sql new file mode 100644 index 0000000..2f3469b --- /dev/null +++ b/models/staging/edfi_3/base/base_ef3__survey_program_associations.sql @@ -0,0 +1,26 @@ +with survey_question_responses as ( + {{ source_edfi3('survey_question_responses') }} +), +renamed as ( + select + tenant_code, + api_year, + pull_timestamp, + last_modified_timestamp, + file_row_number, + filename, + is_deleted, + + v:id::string as record_guid, + v:programReference:educationOrganizationId::integer as education_organization_id, + v:programReference:programName::string as program_name, + v:programReference:programTypeDescriptor::string as program_type_descriptor, + v:surveyReference:namespace::string as namespace, + v:surveyReference:surveyIdentifier::string as survey_identifier, + -- references + v:programReference as program_reference, + v:surveyReference as survey_reference + + from survey_question_responses +) +select * from renamed diff --git a/models/staging/edfi_3/base/base_ef3__survey_question_responses.sql b/models/staging/edfi_3/base/base_ef3__survey_question_responses.sql index b7ecc2b..55f92d4 100644 --- a/models/staging/edfi_3/base/base_ef3__survey_question_responses.sql +++ b/models/staging/edfi_3/base/base_ef3__survey_question_responses.sql @@ -18,7 +18,7 @@ renamed as ( v:surveyResponseReference:surveyResponseIdentifier::string as survey_response_id, v:comment::string as comment, v:noResponse::boolean as no_response, - --references + -- references v:surveyQuestionReference as survey_question_reference, v:surveyResponseReference as survey_response_reference, -- lists diff --git a/models/staging/edfi_3/base/base_ef3__survey_questions.sql b/models/staging/edfi_3/base/base_ef3__survey_questions.sql index 7002436..ab7e2bf 100644 --- a/models/staging/edfi_3/base/base_ef3__survey_questions.sql +++ b/models/staging/edfi_3/base/base_ef3__survey_questions.sql @@ -19,7 +19,7 @@ renamed as ( v:questionText::string as question_text, -- descriptors {{ extract_descriptor('v:questionFormDescriptor::string') }} as question_form, - --references + -- references v:surveyReference as survey_reference, v:surveySectionReference as survey_section_reference, -- lists diff --git a/models/staging/edfi_3/base/base_ef3__survey_response_education_organization_target_associations_v2.sql b/models/staging/edfi_3/base/base_ef3__survey_response_education_organization_target_associations_v2.sql new file mode 100644 index 0000000..5508450 --- /dev/null +++ b/models/staging/edfi_3/base/base_ef3__survey_response_education_organization_target_associations_v2.sql @@ -0,0 +1,24 @@ +with survey_response_education_organization_target_associations as ( + {{ source_edfi3('survey_response_education_organization_target_associations') }} +), +renamed as ( + select + tenant_code, + api_year, + pull_timestamp, + last_modified_timestamp, + file_row_number, + filename, + is_deleted, + + v:id::string as record_guid, + v:educationOrganizationId:educationOrganizationId::string as education_organization_id, + v:surveyResponseReference:surveyIdentifier::string as survey_id, + v:surveyResponseReference:surveyResponseIdentifier::string as survey_response_id, + -- references + v:educationOrganizationReference as educationOrganizationReference, + v:surveyResponseReference as surveyResponseReference, + + from survey_response_education_organization_target_associations +) +select * from renamed diff --git a/models/staging/edfi_3/base/base_ef3__survey_section_associations.sql b/models/staging/edfi_3/base/base_ef3__survey_section_associations.sql new file mode 100644 index 0000000..567554c --- /dev/null +++ b/models/staging/edfi_3/base/base_ef3__survey_section_associations.sql @@ -0,0 +1,24 @@ +with survey_section_associations as ( + {{ source_edfi3('survey_section_associations') }} +), +renamed as ( + select + tenant_code, + api_year, + pull_timestamp, + last_modified_timestamp, + file_row_number, + filename, + is_deleted, + + v:id::string as record_guid, + v:sectionReference::sectionIdentifier::string as section_id, + v:sectionReference::sessionName::string as session_name, + v:sectionReference::schoolId::string as school_id, + v:sectionReference::schoolYear::string as school_year, + -- references + v:sectionReference as sectionReference + v:surveyReference as surveyReference + from survey_section_response_education_organization_target_associations +) +select * from renamed diff --git a/models/staging/edfi_3/base/base_ef3__survey_section_response_education_organization_target_associations.sql b/models/staging/edfi_3/base/base_ef3__survey_section_response_education_organization_target_associations.sql new file mode 100644 index 0000000..b51a0ea --- /dev/null +++ b/models/staging/edfi_3/base/base_ef3__survey_section_response_education_organization_target_associations.sql @@ -0,0 +1,22 @@ +with survey_section_response_education_organization_target_associations as ( + {{ source_edfi3('survey_section_response_education_organization_target_associations') }} +), +renamed as ( + select + tenant_code, + api_year, + pull_timestamp, + last_modified_timestamp, + file_row_number, + filename, + is_deleted, + + v:id::string as record_guid, + v:surveySectionResponseReference::surveyIdentifier::string as survey_id, + v:surveySectionResponseReference::surveySectionTitle::string as survey_section_title, + -- references + v:educationOrganizationReference as educationOrganizationReference + v:surveySectionResponseReference as surveySectionResponseReference + from survey_section_response_education_organization_target_associations +) +select * from renamed diff --git a/models/staging/tpdm/base/base_tpdm__survey_section_aggregate_responses.sql b/models/staging/edfi_3/base/base_ef3__survey_section_response_staff_target_associations.sql similarity index 53% rename from models/staging/tpdm/base/base_tpdm__survey_section_aggregate_responses.sql rename to models/staging/edfi_3/base/base_ef3__survey_section_response_staff_target_associations.sql index c48bea1..8a04c58 100644 --- a/models/staging/tpdm/base/base_tpdm__survey_section_aggregate_responses.sql +++ b/models/staging/edfi_3/base/base_ef3__survey_section_response_staff_target_associations.sql @@ -10,7 +10,12 @@ renamed as ( file_row_number, filename, is_deleted, - ?? + + v:id::string as record_guid, + v:surveySectionResponseReference::surveyIdentifier::string as survey_id, + v:surveySectionResponseReference::surveySectionTitle::string as survey_section_title, + -- references + v:staffReference as staff_reference from survey_section_aggregate_responses ) select * from renamed diff --git a/models/staging/tpdm/base/base_tpdm__persons.sql b/models/staging/tpdm/base/base_tpdm__persons.sql deleted file mode 100644 index 4937abc..0000000 --- a/models/staging/tpdm/base/base_tpdm__persons.sql +++ /dev/null @@ -1,16 +0,0 @@ -with persons as ( - {{ source_edfi3('persons') }} -), -renamed as ( - select - tenant_code, - api_year, - pull_timestamp, - last_modified_timestamp, - filename, - file_row_number, - is_deleted, - ??? - from persons -) -select * from renamed diff --git a/models/staging/tpdm/base/base_tpdm__staff_educator_preparation_program_associations.sql b/models/staging/tpdm/base/base_tpdm__staff_educator_preparation_program_associations.sql new file mode 100644 index 0000000..3fa70eb --- /dev/null +++ b/models/staging/tpdm/base/base_tpdm__staff_educator_preparation_program_associations.sql @@ -0,0 +1,36 @@ +with + staff_educator_preparation_program_associations as ( + {{ source_edfi3('staff_educator_preparation_program_associations') }} + ), + renamed as ( + select + tenant_code, + api_year, + pull_timestamp, + last_modified_timestamp, + file_row_number, + filename, + is_deleted, + + v:id::string as record_guid, + v:begindate::date as begin_date, + v:enddate::date as end_date, + -- course offering key + v:candidatereference:candidateidentifier::integer as candidate_identifier, + -- location key + v:educatorpreparationprogramreference:educationorganizationid::integer as classroom_identification_code, + v:educatorpreparationprogramreference:programname::string as classroom_location_school_id, + v:educatorpreparationprogramreference:programtypedescriptor::string as classroom_location_school_id, + -- descriptors + {{ extract_descriptor('v:eppProgramPathwayDescriptor::string') }} as epp_program_pathway_descriptor, + {{ extract_descriptor('v:reasonExitedDescriptor::string') }} as reason_exited_descriptor, + -- references + v:candidatereference as candidate_reference, + v:educatorpreparationprogramreference as educator_preparation_program_reference, + -- lists + v:cohortyears as v_cohort_years, + v:degreespecializations as v_degree_specializations + + from staff_educator_preparation_program_associations + ) +select * from renamed diff --git a/models/staging/tpdm/base/base_tpdm__survey_responses.sql b/models/staging/tpdm/base/base_tpdm__survey_responses.sql index 1d250ba..7b9cc8d 100644 --- a/models/staging/tpdm/base/base_tpdm__survey_responses.sql +++ b/models/staging/tpdm/base/base_tpdm__survey_responses.sql @@ -21,7 +21,7 @@ renamed as ( v:location::string as location, v:responseDate::date as response_date, v:responseTime::int as completion_time_seconds, - --references + -- references v:surveyReference as survey_reference, v:studentReference as student_reference, v:staffReference as staff_reference, diff --git a/models/staging/tpdm/base/base_tpdm__survey_section_response_person_target_associations.sql b/models/staging/tpdm/base/base_tpdm__survey_section_response_person_target_associations.sql new file mode 100644 index 0000000..6107524 --- /dev/null +++ b/models/staging/tpdm/base/base_tpdm__survey_section_response_person_target_associations.sql @@ -0,0 +1,23 @@ +with survey_section_response_person_target_associations as ( + {{ source_edfi3('survey_section_response_person_target_associations') }} +), +renamed as ( + select + tenant_code, + api_year, + pull_timestamp, + last_modified_timestamp, + file_row_number, + filename, + is_deleted, + + v:id::string as record_guid, + v:personReference:personId::string as person_id, + v:surveySectionResponseReference:surveyIdentifier::string as survey_id, + v:surveySectionResponseReference:surveyResponseIdentifier::string as survey_response_id, + -- references + v:surveySectionResponseReference as survey_section_response_reference, + v:personReference as person_reference + from survey_response_person_target_associations +) +select * from renamed diff --git a/models/staging/tpdm/base/base_tpdm__survey_section_responses.sql b/models/staging/tpdm/base/base_tpdm__survey_section_responses.sql deleted file mode 100644 index 78bb06f..0000000 --- a/models/staging/tpdm/base/base_tpdm__survey_section_responses.sql +++ /dev/null @@ -1,23 +0,0 @@ -with section_responses as ( - {{ source_edfi3('section_responses') }} -), -renamed as ( - select - tenant_code, - api_year, - pull_timestamp, - last_modified_timestamp, - file_row_number, - filename, - is_deleted, - - v:id::string as record_guid, - v:surveyResponseReference:surveyResponseIdentifier::string as survey_response_id, - v:surveySectionReference:surveyIdentifier::string as survey_id, - v:sectionRating::decimal(9,3) as section_rating, - -- references - v:surveyResponseReference as survey_response_reference, - v:surveySectionReference as survey_section_reference - from section_responses -) -select * from renamed diff --git a/models/staging/tpdm/base/base_tpdm__survey_sections.sql b/models/staging/tpdm/base/base_tpdm__survey_sections.sql deleted file mode 100644 index 9b40ea5..0000000 --- a/models/staging/tpdm/base/base_tpdm__survey_sections.sql +++ /dev/null @@ -1,21 +0,0 @@ -with survey_sections as ( - {{ source_edfi3('survey_sections') }} -), -renamed as ( - select - tenant_code, - api_year, - pull_timestamp, - last_modified_timestamp, - file_row_number, - filename, - is_deleted, - - v:id::string as record_guid, - v:surveyReference::surveyIdentifier::string as survey_id, - v:surveySectionTitle::string as survey_section_title, - -- references - v:surveyReference as survey_reference - from survey_sections -) -select * from renamed From c28d086684695153f0f83024e9e59ebad5ba54c1 Mon Sep 17 00:00:00 2001 From: Vasili Zakharko Date: Wed, 24 Jul 2024 20:31:35 +0300 Subject: [PATCH 3/3] feature: create base tpdm/edfi_3 models iteration 2 --- models/staging/edfi_3/base/_edfi_3__base.yml | 68 ++++++++++++++++++- .../base/base_ef3__rubric_dimensions.sql | 16 ----- ...on_organization_target_associations_v2.sql | 24 ------- models/staging/tpdm/base/_tpdm__base.yml | 25 +------ .../staging/tpdm/base/base_tpdm__schools.sql | 52 -------------- .../staging/tpdm/base/base_tpdm__sections.sql | 52 -------------- .../base_tpdm__student_academic_records.sql | 59 ---------------- .../staging/tpdm/base/base_tpdm__students.sql | 43 ------------ .../tpdm/base/base_tpdm__survey_responses.sql | 35 ---------- ...on_response_person_target_associations.sql | 2 +- 10 files changed, 70 insertions(+), 306 deletions(-) delete mode 100644 models/staging/edfi_3/base/base_ef3__rubric_dimensions.sql delete mode 100644 models/staging/edfi_3/base/base_ef3__survey_response_education_organization_target_associations_v2.sql delete mode 100644 models/staging/tpdm/base/base_tpdm__schools.sql delete mode 100644 models/staging/tpdm/base/base_tpdm__sections.sql delete mode 100644 models/staging/tpdm/base/base_tpdm__student_academic_records.sql delete mode 100644 models/staging/tpdm/base/base_tpdm__students.sql delete mode 100644 models/staging/tpdm/base/base_tpdm__survey_responses.sql diff --git a/models/staging/edfi_3/base/_edfi_3__base.yml b/models/staging/edfi_3/base/_edfi_3__base.yml index 859f5a3..4115f66 100644 --- a/models/staging/edfi_3/base/_edfi_3__base.yml +++ b/models/staging/edfi_3/base/_edfi_3__base.yml @@ -91,24 +91,45 @@ models: - name: base_ef3__programs config: tags: ['core'] - - name: base_ef3__rubric_dimensions + - name: base_ef3__school_year_types config: tags: ['core'] - name: base_ef3__schools config: tags: ['core'] + - name: base_ef3__section_attendance_taken_events + config: + tags: ['core'] - name: base_ef3__sections config: tags: ['core'] - name: base_ef3__sessions config: tags: ['core'] + - name: base_ef3__staff_absence_events + config: + tags: ['core'] + - name: base_ef3__staff_cohort_associations + config: + tags: ['core'] + - name: base_ef3__staff_discipline_incident_associations + config: + tags: ['core'] - name: base_ef3__staff_education_organization_assignment_associations config: tags: ['core'] + - name: base_ef3__staff_education_organization_contact_associations + config: + tags: ['core'] - name: base_ef3__staff_education_organization_employment_associations config: tags: ['core'] + - name: base_ef3__staff_leaves + config: + tags: ['core'] + - name: base_ef3__staff_program_associations + config: + tags: ['core'] - name: base_ef3__staff_school_associations config: tags: ['core'] @@ -129,6 +150,12 @@ models: config: tags: ['cohort'] enabled: "{{ var('src:domain:cohort:enabled', True) }}" + - name: base_ef3__student_competency_objectives + config: + tags: ['base'] + - name: base_ef3__student_cte_program_associations + config: + tags: ['base'] - name: base_ef3__student_discipline_incident_associations config: tags: ['discipline'] @@ -144,26 +171,50 @@ models: - name: base_ef3__student_education_organization_associations config: tags: ['core'] + - name: base_ef3__student_education_organization_responsibility_associations + config: + tags: ['core'] + - name: base_ef3__student_gradebook_entries + config: + tags: ['core'] - name: base_ef3__student_homeless_program_associations config: tags: ['homeless'] enabled: "{{ var('src:program:homeless:enabled', True) }}" + - name: base_ef3__student_intervention_associations + config: + tags: ['core'] + - name: base_ef3__student_intervention_attendance_events + config: + tags: ['core'] - name: base_ef3__student_language_instruction_program_associations config: tags: ['language_instruction'] enabled: "{{ var('src:program:language_instruction:enabled', True) }}" + - name: base_ef3__student_migrant_education_program_associations + config: + tags: ['core'] + - name: base_ef3__student_neglected_or_delinquent_program_associations + config: + tags: ['core'] - name: base_ef3__student_parent_associations config: tags: ['core'] - name: base_ef3__student_program_associations config: tags: ['core'] + - name: base_ef3__student_program_attendance_events + config: + tags: ['core'] - name: base_ef3__student_school_associations config: tags: ['core'] - name: base_ef3__student_school_attendance_events config: tags: ['core'] + - name: base_ef3__student_school_food_service_program_associations + config: + tags: ['core'] - name: base_ef3__student_section_associations config: tags: ['core'] @@ -181,6 +232,12 @@ models: - name: base_ef3__students config: tags: ['core'] + - name: base_ef3__survey_course_associations + config: + tags: ['survey'] + - name: base_ef3__survey_program_associations + config: + tags: ['survey'] - name: base_ef3__survey_question_responses config: tags: ['survey'] @@ -209,6 +266,15 @@ models: - name: base_ef3__survey_section_aggregate_responses config: tags: ['survey'] + - name: base_ef3__survey_section_associations + config: + tags: ['survey'] + - name: base_ef3__survey_section_response_education_organization_target_associations + config: + tags: ['survey'] + - name: base_ef3__survey_section_response_staff_target_associations + config: + tags: ['survey'] - name: base_ef3__survey_section_responses config: tags: ['survey'] diff --git a/models/staging/edfi_3/base/base_ef3__rubric_dimensions.sql b/models/staging/edfi_3/base/base_ef3__rubric_dimensions.sql deleted file mode 100644 index e39905c..0000000 --- a/models/staging/edfi_3/base/base_ef3__rubric_dimensions.sql +++ /dev/null @@ -1,16 +0,0 @@ -with rubric_dimensions as ( - {{ source_edfi3('rubric_dimensions') }} -), -renamed as ( - select - tenant_code, - api_year, - pull_timestamp, - last_modified_timestamp, - filename, - file_row_number, - is_deleted, - ??? - from rubric_dimensions -) -select * from renamed diff --git a/models/staging/edfi_3/base/base_ef3__survey_response_education_organization_target_associations_v2.sql b/models/staging/edfi_3/base/base_ef3__survey_response_education_organization_target_associations_v2.sql deleted file mode 100644 index 5508450..0000000 --- a/models/staging/edfi_3/base/base_ef3__survey_response_education_organization_target_associations_v2.sql +++ /dev/null @@ -1,24 +0,0 @@ -with survey_response_education_organization_target_associations as ( - {{ source_edfi3('survey_response_education_organization_target_associations') }} -), -renamed as ( - select - tenant_code, - api_year, - pull_timestamp, - last_modified_timestamp, - file_row_number, - filename, - is_deleted, - - v:id::string as record_guid, - v:educationOrganizationId:educationOrganizationId::string as education_organization_id, - v:surveyResponseReference:surveyIdentifier::string as survey_id, - v:surveyResponseReference:surveyResponseIdentifier::string as survey_response_id, - -- references - v:educationOrganizationReference as educationOrganizationReference, - v:surveyResponseReference as surveyResponseReference, - - from survey_response_education_organization_target_associations -) -select * from renamed diff --git a/models/staging/tpdm/base/_tpdm__base.yml b/models/staging/tpdm/base/_tpdm__base.yml index e603a7e..9ee6a6e 100644 --- a/models/staging/tpdm/base/_tpdm__base.yml +++ b/models/staging/tpdm/base/_tpdm__base.yml @@ -37,39 +37,18 @@ models: - name: base_tpdm__performance_evaluations config: tags: ['tpdm'] - - name: base_tpdm__persons - config: - tags: ['tpdm'] - name: base_tpdm__post_secondary_institutions config: tags: ['tpdm'] - name: base_tpdm__rubric_dimensions config: tags: ['tpdm'] - - name: base_tpdm__schools - config: - tags: ['tpdm'] - - name: base_tpdm__sections - config: - tags: ['tpdm'] - - name: base_tpdm__student_academic_records - config: - tags: ['tpdm'] - - name: base_tpdm__students + - name: base_tpdm__staff_educator_preparation_program_associations config: tags: ['tpdm'] - name: base_tpdm__survey_response_person_target_associations config: tags: ['tpdm'] - - name: base_tpdm__survey_responses - config: - tags: ['tpdm'] - - name: base_tpdm__survey_section_aggregate_responses - config: - tags: ['tpdm'] - - name: base_tpdm__survey_section_responses - config: - tags: ['tpdm'] - - name: base_tpdm__survey_sections + - name: base_tpdm__survey_section_response_person_target_associations config: tags: ['tpdm'] diff --git a/models/staging/tpdm/base/base_tpdm__schools.sql b/models/staging/tpdm/base/base_tpdm__schools.sql deleted file mode 100644 index a1ba275..0000000 --- a/models/staging/tpdm/base/base_tpdm__schools.sql +++ /dev/null @@ -1,52 +0,0 @@ -with schools as ( - {{ source_edfi3('schools') }} -), -renamed as ( - select - tenant_code, - api_year, - pull_timestamp, - last_modified_timestamp, - file_row_number, - filename, - is_deleted, - v:id::string as record_guid, - v:schoolId::integer as school_id, - v:nameOfInstitution::string as school_name, - v:shortNameOfInstitution::string as school_short_name, - v:webSite::string as website, - v:localEducationAgencyReference:localEducationAgencyId as lea_id, - -- pull out school categories - case - when array_size(v:schoolCategories) = 1 - then {{ extract_descriptor('v:schoolCategories[0]:schoolCategoryDescriptor::string') }} - when array_size(v:schoolCategories) > 1 - then 'Multiple Categories' - else NULL - end as school_category, - -- descriptors - {{ extract_descriptor('v:schoolTypeDescriptor::string') }} as school_type, - {{ extract_descriptor('v:operationalStatusDescriptor::string') }} as operational_status, - {{ extract_descriptor('v:administrativeFundingControlDescriptor::string') }} as administrative_funding_control, - {{ extract_descriptor('v:internetAccessDescriptor::string') }} as internet_access, - {{ extract_descriptor('v:titleIPartASchoolDesignationDescriptor::string') }} as title_i_part_a_school_designation, - {{ extract_descriptor('v:charterStatusDescriptor::string') }} as charter_status, - {{ extract_descriptor('v:charterApprovalAgencyTypeDescriptor::string') }} as charter_approval_agency, - {{ extract_descriptor('v:magnetSpecialProgramEmphasisSchoolDescriptor::string') }} as magnet_type, - -- references - v:localEducationAgencyReference as local_education_agency_reference, - -- unflattened lists - v:addresses as v_addresses, - v:educationOrganizationCategories as v_education_organization_categories, - v:gradeLevels as v_grade_levels, - v:identificationCodes as v_identification_codes, - v:indicators as v_indicators, - v:institutionTelephones as v_institution_telephones, - v:internationalAddresses as v_international_addresses, - v:schoolCategories as v_school_categories, - - -- edfi extensions - v:_ext as v_ext - from schools -) -select * from renamed diff --git a/models/staging/tpdm/base/base_tpdm__sections.sql b/models/staging/tpdm/base/base_tpdm__sections.sql deleted file mode 100644 index 7483977..0000000 --- a/models/staging/tpdm/base/base_tpdm__sections.sql +++ /dev/null @@ -1,52 +0,0 @@ -with sections as ( - {{ source_edfi3('sections') }} -), -renamed as ( - select - tenant_code, - api_year, - pull_timestamp, - last_modified_timestamp, - file_row_number, - filename, - is_deleted, - v:id::string as record_guid, - v:sectionIdentifier::string as section_id, - v:sectionName::string as section_name, - -- course offering key - v:courseOfferingReference:schoolId::integer as school_id, - v:courseOfferingReference:localCourseCode::string as local_course_code, - v:courseOfferingReference:sessionName::string as session_name, - v:courseOfferingReference:schoolYear::integer as school_year, - -- location key - v:locationReference:classroomIdentificationCode::string as classroom_identification_code, - v:locationReference:schoolId::integer as classroom_location_school_id, - -- location school key - v:locationSchoolReference:schoolId::integer as school_location_school_id, - -- values - v:availableCreditConversion::float as available_credit_conversion, - v:availableCredits::float as available_credits, - v:sequenceOfCourse::integer as sequence_of_course, - v:officialAttendancePeriod::boolean as is_official_attendance_period, - -- descriptors - {{ extract_descriptor('v:availableCreditTypeDescriptor::string') }} as available_credit_type, - {{ extract_descriptor('v:educationalEnvironmentDescriptor::string') }} as educational_environment_type, - {{ extract_descriptor('v:instructionLanguageDescriptor::string') }} as instruction_language, - {{ extract_descriptor('v:mediumOfInstructionDescriptor::string') }} as medium_of_instruction, - {{ extract_descriptor('v:populationServedDescriptor::string') }} as population_served, - -- references - v:courseOfferingReference as course_offering_reference, - v:locationReference as location_reference, - v:locationSchoolReference as location_school_reference, - -- lists - v:characteristics as v_section_characteristics, - v:classPeriods as v_class_periods, - v:courseLevelCharacteristics as v_course_level_characteristics, - v:offeredGradeLevels as v_offered_grade_levels, - v:programs as v_programs, - - -- edfi extensions - v:_ext as v_ext - from sections -) -select * from renamed diff --git a/models/staging/tpdm/base/base_tpdm__student_academic_records.sql b/models/staging/tpdm/base/base_tpdm__student_academic_records.sql deleted file mode 100644 index f83df83..0000000 --- a/models/staging/tpdm/base/base_tpdm__student_academic_records.sql +++ /dev/null @@ -1,59 +0,0 @@ -with academic_records as ( - {{ source_edfi3('student_academic_records') }} -), -renamed as ( - select - tenant_code, - api_year, - pull_timestamp, - last_modified_timestamp, - file_row_number, - filename, - is_deleted, - v:id::string as record_guid, - -- identity components - v:educationOrganizationReference:educationOrganizationId::int as ed_org_id, - v:educationOrganizationReference:link:rel::string as ed_org_type, - v:schoolYearTypeReference:schoolYear::int as school_year, - v:studentReference:studentUniqueId::string as student_unique_id, - {{ extract_descriptor('v:termDescriptor::string') }} as academic_term, - -- non-identity components - v:cumulativeAttemptedCreditConversion::float as cumulative_attempted_credit_conversion, - v:cumulativeAttemptedCredits::float as cumulative_attempted_credits, - v:cumulativeEarnedCreditConversion::float as cumulative_earned_credit_conversion, - v:cumulativeEarnedCredits::float as cumulative_earned_credits, - v:projectedGraduationDate::date as projected_graduation_date, - v:sessionAttemptedCreditConversion::float as session_attempted_credit_conversion, - v:sessionAttemptedCredits::float as session_attempted_credits, - v:sessionEarnedCreditConversion::float as session_earned_credit_conversion, - v:sessionEarnedCredits::float as session_earned_credits, - v:classRanking:classRank::float as class_rank, - v:classRanking:totalNumberInClass::float as class_rank_total_students, - v:classRanking:percentageRanking::float as class_percent_rank, - v:classRanking:classRankingDate::date as class_rank_date, - -- deprecated components (use gpa list instead) - v:cumulativeGradePointAverage::float as cumulative_gpa, - v:cumulativeGradePointsEarned::float as cumulative_grade_points_earned, - v:gradeValueQualifier::string as grade_value_qualifier, - v:sessionGradePointAverage::float as session_gpa, - v:sessionGradePointsEarned::float as session_grade_points_earned, - -- descriptors - {{ extract_descriptor('v:sessionEarnedCreditTypeDescriptor::string') }} as session_earned_credit_type, - {{ extract_descriptor('v:sessionAttemptedCreditTypeDescriptor::string') }} as session_attempted_credit_type, - {{ extract_descriptor('v:cumulativeEarnedCreditTypeDescriptor::string') }} as cumulative_earned_credit_type, - {{ extract_descriptor('v:cumulativeAttemptedCreditTypeDescriptor::string') }} as cumulative_attempted_credit_type, - -- references - v:studentReference as student_reference, - v:educationOrganizationReference as education_organization_reference, - -- lists - v:academicHonors as v_academic_honors, - v:diplomas as v_diplomas, - v:gradePointAverages as v_grade_point_averages, - v:recognitions as v_recognitions, - v:reportCards as v_report_cards, - - -- edfi extensions - v:_ext as v_ext - from academic_records -) -select * from renamed diff --git a/models/staging/tpdm/base/base_tpdm__students.sql b/models/staging/tpdm/base/base_tpdm__students.sql deleted file mode 100644 index ad32299..0000000 --- a/models/staging/tpdm/base/base_tpdm__students.sql +++ /dev/null @@ -1,43 +0,0 @@ -with students as ( - {{ source_edfi3('students') }} -), -renamed as ( - select - tenant_code, - api_year, - pull_timestamp, - last_modified_timestamp, - file_row_number, - filename, - is_deleted, - v:id::string as record_guid, - v:studentUniqueId::string as student_unique_id, - v:firstName::string as first_name, - v:middleName::string as middle_name, - v:lastSurname::string as last_name, - v:generationCodeSuffix::string as generation_code_suffix, - v:maidenName::string as maiden_name, - v:personalTitlePrefix::string as personal_title_prefix, - v:personReference:personId::string as person_id, - v:birthDate::date as birth_date, - v:birthCity::string as birth_city, - v:birthInternationalProvince::string as birth_international_province, - v:dateEnteredUS::date as date_entered_us, - v:multipleBirthStatus::boolean as is_multiple_birth, - -- descriptors - {{ extract_descriptor('v:personReference:sourceSystemDescriptor::string') }} as person_source_system, - {{ extract_descriptor('v:birthSexDescriptor::string') }} as birth_sex, - {{ extract_descriptor('v:citizenshipStatusDescriptor::string') }} as citizenship_status, - {{ extract_descriptor('v:birthStateAbbreviationDescriptor::string') }} as birth_state, - {{ extract_descriptor('v:birthCountryDescriptor::string') }} as birth_country, - -- nested lists - v:identificationDocuments as v_identification_documents, - v:otherNames as v_other_names, - v:personalIdentificationDocuments as v_personal_identification_documents, - v:visas as v_visas, - - -- edfi extensions - v:_ext as v_ext - from students -) -select * from renamed diff --git a/models/staging/tpdm/base/base_tpdm__survey_responses.sql b/models/staging/tpdm/base/base_tpdm__survey_responses.sql deleted file mode 100644 index 7b9cc8d..0000000 --- a/models/staging/tpdm/base/base_tpdm__survey_responses.sql +++ /dev/null @@ -1,35 +0,0 @@ -with survey_responses as ( - {{ source_edfi3('survey_responses') }} -), -renamed as ( - select - tenant_code, - api_year, - pull_timestamp, - last_modified_timestamp, - file_row_number, - filename, - is_deleted, - - v:id::string as record_guid, - v:surveyReference:surveyIdentifier::string as survey_id, - v:surveyReference:namespace::string as namespace, - v:surveyResponseIdentifier::string as survey_response_id, - v:studentReference:studentUniqueId::string as student_unique_id, - v:electronicMailAddress::string as electronic_mail_address, - v:fullName::string as full_name, - v:location::string as location, - v:responseDate::date as response_date, - v:responseTime::int as completion_time_seconds, - -- references - v:surveyReference as survey_reference, - v:studentReference as student_reference, - v:staffReference as staff_reference, - v:parentReference as parent_reference, - -- lists - v:surveyLevels as v_survey_levels, - -- edfi extensions - v:_ext as v_ext - from survey_responses -) -select * from renamed diff --git a/models/staging/tpdm/base/base_tpdm__survey_section_response_person_target_associations.sql b/models/staging/tpdm/base/base_tpdm__survey_section_response_person_target_associations.sql index 6107524..59afbc5 100644 --- a/models/staging/tpdm/base/base_tpdm__survey_section_response_person_target_associations.sql +++ b/models/staging/tpdm/base/base_tpdm__survey_section_response_person_target_associations.sql @@ -17,7 +17,7 @@ renamed as ( v:surveySectionResponseReference:surveyResponseIdentifier::string as survey_response_id, -- references v:surveySectionResponseReference as survey_section_response_reference, - v:personReference as person_reference + v:personReference as person_reference from survey_response_person_target_associations ) select * from renamed