Merge pull request #17 from Unicon/feature/create-stage-evaluation-mo…

…dels Create stage TPDM models
Unicon · Jul 31, 2024 · 451d3ef · 451d3ef
2 parents 0c3bd26 + 8eed385
commit 451d3ef
Show file tree

Hide file tree

Showing 29 changed files with 907 additions and 83 deletions.
diff --git a/macros/gen_skey.sql b/macros/gen_skey.sql
@@ -201,7 +201,102 @@
             'col_list': ['candidateIdentifier'],
             'annualize': True
         },
-
+        'k_person': {
+            'reference_name': 'person_reference',
+            'col_list': ['personId', 'sourceSystemDescriptor'],
+            'annualize': True
+        },
+        'k_educator_prep_program': {
+            'reference_name': 'educator_preparation_program_reference',
+            'col_list': ['educationOrganizationId',
+                         'programName',
+                         'programTypeDescriptor'],
+            'annualize': True
+        },
+        'k_performance_evaluation': {
+            'reference_name': 'performance_evaluation_reference',
+            'col_list': ['educationOrganizationId', 
+                         'performanceEvaluationTitle', 
+                         'schoolYear'],
+            'annualize': True
+        },
+        'k_evaluation': {
+            'reference_name': 'evaluation_reference',
+            'col_list': ['educationOrganizationId',
+                         'evaluationPeriodDescriptor',
+                         'evaluationTitle',
+                         'performanceEvaluationTitle',
+                         'performanceEvaluationTypeDescriptor',
+                         'schoolYear',
+                         'termDescriptor'],
+            'annualize': True
+        },
+        'k_evaluation_rating': {
+            'reference_name': 'evaluation_rating_reference',
+            'col_list': ['educationOrganizationId',
+                         'evaluationDate',
+                         'evaluationPeriodDescriptor',
+                         'evaluationTitle',
+                         'performanceEvaluationTitle',
+                         'performanceEvaluationTypeDescriptor',
+                         'personId',
+                         'schoolYear',
+                         'sourceSystemDescriptor',
+                         'termDescriptor'],
+            'annualize': True
+        },
+        'k_evaluation_objective': {
+            'reference_name': 'evaluation_objective_reference',
+            'col_list': ['educationOrganizationId',
+                         'evaluationObjectiveTitle',
+                         'evaluationPeriodDescriptor',
+                         'evaluationTitle',
+                         'performanceEvaluationTitle',
+                         'performanceEvaluationTypeDescriptor',
+                         'schoolYear',
+                         'termDescriptor'],
+            'annualize': True
+        },
+        'k_evaluation_element': {
+            'reference_name': 'evaluation_element_reference',
+            'col_list': ['educationOrganizationId',
+                         'evaluationElementTitle',
+                         'evaluationObjectiveTitle',
+                         'evaluationPeriodDescriptor',
+                         'evaluationTitle',
+                         'performanceEvaluationTitle',
+                         'performanceEvaluationTypeDescriptor',
+                         'schoolYear',
+                         'termDescriptor'],
+            'annualize': True
+        },
+        'k_evaluation_objective_rating': {
+            'reference_name': 'evaluation_objective_rating_reference',
+            'col_list': ['educationOrganizationId',
+                         'evaluationDate',
+                         'evaluationObjectiveTitle',
+                         'evaluationPeriodDescriptor',
+                         'evaluationTitle',
+                         'performanceEvaluationTitle',
+                         'performanceEvaluationTypeDescriptor',
+                         'personId',
+                         'schoolYear',
+                         'sourceSystemDescriptor',
+                         'termDescriptor'],
+            'annualize': True
+        },
+        'k_performance_evaluation_rating': {
+            'reference_name': 'performance_evaluation_rating_reference',
+            'col_list': ['educationOrganizationId',
+                         'evaluationPeriodDescriptor',
+                         'performanceEvaluationTitle',
+                         'performanceEvaluationTypeDescriptor',
+                         'personId',
+                         'sourceSystemDescriptor',
+                         'schoolYear',
+                         'termDescriptor'],
+            'annualize': True
+        },
         'k_template': {
             'reference_name': '',
             'col_list': [],

diff --git a/models/staging/edfi_3/stage/_edfi_3__stage.yml b/models/staging/edfi_3/stage/_edfi_3__stage.yml
@@ -114,17 +114,25 @@ referential_integrity_tests:
         to: ref('stg_ef3__surveys')
         field: k_survey
         tags: ['ref_integrity']
+
   - k_survey_question: &ref_k_survey_question
     - relationships:
         to: ref('stg_ef3__survey_questions')
         field: k_survey_question
         tags: ['ref_integrity']
+
   - k_survey_response: &ref_k_survey_response
     - relationships:
         to: ref('stg_ef3__survey_responses')
         field: k_survey_response
         tags: ['ref_integrity']
 
+  - k_person: &ref_k_person
+    - relationships:
+        to: ref('stg_ef3__people')
+        field: k_person
+        tags: ['ref_integrity']
+
 
 models:
   - name: stg_ef3__assessments

diff --git a/models/staging/tpdm/base/base_tpdm__candidate_educator_preparation_program_associations.sql b/models/staging/tpdm/base/base_tpdm__candidate_educator_preparation_program_associations.sql
@@ -11,12 +11,16 @@ renamed as (
         filename,
         is_deleted,
 
-        v:id::string                                                        as record_guid,
-        v:candidateReference:candidateIdentifier::string                    as candidate_id,
-        v:educatorPreparationProgramReference:educationOrganizationId::int  as ed_org_id,
-        v:educatorPreparationProgramReference:link:rel::string              as ed_org_type,
-        v:beginDate::date                                                   as begin_date,
-        v:endDate::date                                                     as end_date,
+        v:id::string as record_guid,
+        -- identity components
+        v:beginDate::date                                                                       as begin_date,
+        v:candidateReference:candidateIdentifier::string                                        as candidate_id,
+        v:educatorPreparationProgramReference:educationOrganizationId::int                      as ed_org_id,
+        v:educatorPreparationProgramReference:programName::string                               as program_name,
+        {{ extract_descriptor('v:educatorPreparationProgramReference:programTypeDescriptor') }} as program_type,
+        -- non-identity components
+        v:educatorPreparationProgramReference:link:rel::string as ed_org_type,
+        v:endDate::date                                        as end_date,
         -- descriptors
         {{ extract_descriptor('v:reasonExitedDescriptor::string') }}      as reason_exited,
         {{ extract_descriptor('v:eppProgramPathwayDescriptor::string') }} as epp_program_pathway,
@@ -25,7 +29,9 @@ renamed as (
         v:cohortYears           as v_cohort_years,
         -- references
         v:candidateReference                  as candidate_reference,
-        v:educatorPreparationProgramReference as educator_preparation_program_reference
+        v:educatorPreparationProgramReference as educator_preparation_program_reference,
+        -- edfi extensions
+        v:_ext as v_ext
     from candidate_educator_preparation_program_associations
 )
 select * from renamed
diff --git a/models/staging/tpdm/base/base_tpdm__candidates.sql b/models/staging/tpdm/base/base_tpdm__candidates.sql
@@ -48,9 +48,10 @@ renamed as (
         v:personalIdentificationDocuments as v_personal_identification_documents,
         v:races                           as v_races,
         v:telephones                      as v_telephones,
-
         -- references
-        v:personReference as person_reference
+        v:personReference as person_reference,
+        -- edfi extensions
+        v:_ext as v_ext
     from candidates
 )
 select * from renamed
diff --git a/models/staging/tpdm/base/base_tpdm__evaluation_element_ratings.sql b/models/staging/tpdm/base/base_tpdm__evaluation_element_ratings.sql
@@ -11,7 +11,21 @@ renamed as (
         filename,
         is_deleted,
 
-        v:id::string                  as record_guid,
+        v:id::string as record_guid,
+        -- identity components
+        v:evaluationElementReference:educationOrganizationId::int                                            as ed_org_id,
+        v:evaluationObjectiveRatingReference:evaluationDate::timestamp                                       as evaluation_date,
+        v:evaluationElementReference:evaluationElementTitle::string                                          as evaluation_element_title,
+        v:evaluationElementReference:evaluationObjectiveTitle::string                                        as evaluation_objective_title,
+        {{ extract_descriptor('v:evaluationElementReference:evaluationPeriodDescriptor::string') }}          as evaluation_period,
+        v:evaluationElementReference:evaluationTitle::string                                                 as evaluation_title,
+        v:evaluationElementReference:performanceEvaluationTitle::string                                      as perfomance_evaluation_title,
+        {{ extract_descriptor('v:evaluationElementReference:performanceEvaluationTypeDescriptor::string') }} as performance_evaluation_type,
+        v:evaluationObjectiveRatingReference:personId::string                                                as person_id,
+        v:evaluationElementReference:schoolYear::int                                                         as school_year,
+        {{ extract_descriptor('v:evaluationObjectiveRatingReference:sourceSystemDescriptor::string') }}      as source_system,
+        {{ extract_descriptor('v:evaluationObjectiveRatingReference:termDescriptor::string') }}              as academic_term,
+        -- non-identity components
         v:areaOfRefinement::string    as area_of_refinement,
         v:areaOfReinforcement::string as area_of_reinforcement,
         v:comments::string            as comments,
@@ -22,7 +36,9 @@ renamed as (
         v:elementRatingResults  as v_element_rating_results,
         -- references
         v:evaluationElementReference         as evaluation_element_reference,
-        v:evaluationObjectiveRatingReference as evaluation_objective_rating_reference
+        v:evaluationObjectiveRatingReference as evaluation_objective_rating_reference,
+        -- edfi extensions
+        v:_ext as v_ext
     from evaluation_element_ratings
 )
 select * from renamed
diff --git a/models/staging/tpdm/base/base_tpdm__evaluation_elements.sql b/models/staging/tpdm/base/base_tpdm__evaluation_elements.sql
@@ -11,18 +11,29 @@ renamed as (
         filename,
         is_deleted,
 
-        v:id::string                                                as record_guid,
-        v:evaluationObjectiveReference:educationOrganizationId::int as ed_org_id,
-        v:evaluationElementTitle::string                            as evaluation_element_title,
-        v:sortOrder::int                                            as sort_order,
-        v:minRating::float                                          as min_rating,
-        v:maxRating::float                                          as max_rating,
+        v:id::string as record_guid,
+        -- identity components
+        v:evaluationObjectiveReference:educationOrganizationId::int                                            as ed_org_id,
+        v:evaluationElementTitle::string                                                                       as evaluation_element_title,
+        v:evaluationObjectiveReference:evaluationObjectiveTitle::string                                        as evaluation_objective_title,
+        {{ extract_descriptor('v:evaluationObjectiveReference:evaluationPeriodDescriptor::string') }}          as evaluation_period,
+        v:evaluationObjectiveReference:evaluationTitle::string                                                 as evaluation_title,
+        v:evaluationObjectiveReference:performanceEvaluationTitle::string                                      as performance_evaluation_title,
+        {{ extract_descriptor('v:evaluationObjectiveReference:performanceEvaluationTypeDescriptor::string') }} as performance_evaluation_type,
+        v:evaluationObjectiveReference:schoolYear::int                                                         as school_year,
+        {{ extract_descriptor('v:evaluationObjectiveReference:termDescriptor::string') }}                      as academic_term,
+        -- non-identity components
+        v:sortOrder::int   as sort_order,
+        v:minRating::float as min_rating,
+        v:maxRating::float as max_rating,
         -- descriptors
         {{ extract_descriptor('v:evaluationTypeDescriptor::string') }} as evaluation_type,
         -- unflattened lists
         v:ratingLevels as v_rating_levels,
         -- references
         v:evaluationObjectiveReference as evaluation_objective_reference
+        -- edfi extensions
+        v:_ext as v_ext
     from evaluation_elements
 )
 select * from renamed
diff --git a/models/staging/tpdm/base/base_tpdm__evaluation_objective_ratings.sql b/models/staging/tpdm/base/base_tpdm__evaluation_objective_ratings.sql
@@ -11,16 +11,31 @@ renamed as (
         filename,
         is_deleted,
 
-        v:id::string       as record_guid,
+        v:id::string as record_guid,
+        -- identity components
+        v:evaluationObjectiveReference:educationOrganizationId::int                                            as ed_org_id,
+        v:evaluationRatingReference:evaluationDate::timestamp                                                  as evaluation_date,
+        v:evaluationObjectiveReference:evaluationObjectiveTitle::string                                        as evaluation_objective_title,
+        {{ extract_descriptor('v:evaluationObjectiveReference:evaluationPeriodDescriptor::string') }}          as evaluation_period,
+        v:evaluationObjectiveReference:evaluationTitle::string                                                 as evaluation_title,
+        v:evaluationObjectiveReference:performanceEvaluationTitle::string                                      as perfomance_evaluation_title,
+        {{ extract_descriptor('v:evaluationObjectiveReference:performanceEvaluationTypeDescriptor::string') }} as performance_evaluation_type,
+        v:evaluationRatingReference:personId::string                                                           as person_id,
+        v:evaluationObjectiveReference:schoolYear::int                                                         as school_year,
+        {{ extract_descriptor('v:evaluationRatingReference:sourceSystemDescriptor::string')}}                  as source_system,
+        {{ extract_descriptor('v:evaluationRatingReference:termDescriptor::string')}}                          as academic_term,
+        -- non-identity components
         v:comments::string as comments,
         -- descriptors
         {{ extract_descriptor('v:objectiveRatingLevelDescriptor::string') }} as objective_rating_level,
         -- unflattened lists
-        v:objectiveRatingResults  as v_objective_rating_results,
-        v:results                 as v_results,
+        v:objectiveRatingResults as v_objective_rating_results,
+        v:results                as v_results,
         -- references
         v:evaluationRatingReference    as evaluation_rating_reference,
-        v:evaluationObjectiveReference as evaluation_objective_reference
+        v:evaluationObjectiveReference as evaluation_objective_reference,
+        -- edfi extensions
+        v:_ext as v_ext
     from evaluation_objective_ratings
 )
 select * from renamed
diff --git a/models/staging/tpdm/base/base_tpdm__evaluation_objectives.sql b/models/staging/tpdm/base/base_tpdm__evaluation_objectives.sql
@@ -11,19 +11,29 @@ renamed as (
         filename,
         is_deleted,
 
-        v:id::string                                       as record_guid,
-        v:evaluationReference:educationOrganizationId::int as ed_org_id,
-        v:evaluationObjectiveTitle::string                 as evaluation_objective_title,
-        v:evaluationObjectiveDescription::string           as evaluation_objective_description,
-        v:sortOrder::int                                   as sort_order,
-        v:minRating::float                                 as min_rating,
-        v:maxRating::float                                 as max_rating,
+        v:id::string as record_guid,
+        -- identity components
+        v:evaluationReference:educationOrganizationId::int                                           as ed_org_id,
+        v:evaluationObjectiveTitle::string                                                           as evaluation_objective_title,
+        {{ extract_descriptor('v:evaluationReference:evaluationPeriodDescriptor::string') }}         as evaluation_period,
+        v:evaluationReference:evaluationTitle::string                                                as evaluation_title,
+        v:evaluationReference:performanceEvaluationTitle::string                                     as perfomance_evaluation_title,
+        {{ extract_descriptor('v:evaluationReference:performanceEvaluationTypeDescriptor::string')}} as performance_evaluation_type,
+        v:evaluationReference:schoolYear::int                                                        as school_year,
+        {{ extract_descriptor('v:evaluationReference:termDescriptor::string')}}                      as academic_term,
+        -- non-identity components
+        v:evaluationObjectiveDescription::string as evaluation_objective_description,
+        v:sortOrder::int                         as sort_order,
+        v:minRating::float                       as min_rating,
+        v:maxRating::float                       as max_rating,
         -- descriptors
         {{ extract_descriptor('v:evaluationTypeDescriptor::string') }} as evaluation_type,
         -- unflattened lists
         v:ratingLevels as v_rating_levels,
         -- references
-        v:evaluationReference as evaluation_reference
+        v:evaluationReference as evaluation_reference,
+        -- edfi extensions
+        v:_ext as v_ext
     from evaluation_objectives
 )
 select * from renamed
diff --git a/models/staging/tpdm/base/base_tpdm__evaluation_ratings.sql b/models/staging/tpdm/base/base_tpdm__evaluation_ratings.sql
@@ -11,9 +11,20 @@ renamed as (
         filename,
         is_deleted,
 
-        v:id::string                                  as record_guid,
-        v:sectionReference:sectionIdentifier::string  as section_id,
-        v:evaluationDate::date                        as evaluation_date,
+        v:id::string as record_guid,
+        -- identity components
+        v:evaluationReference:educationOrganizationId::int                                                as ed_org_id,
+        v:evaluationDate::timestamp                                                                       as evaluation_date,
+        {{ extract_descriptor('v:evaluationReference:evaluationPeriodDescriptor::string') }}              as evaluation_period,
+        v:evaluationReference:evaluationTitle::string                                                     as evaluation_title,
+        v:evaluationReference:performanceEvaluationTitle::string                                          as performance_evaluation_title,
+        {{ extract_descriptor('v:evaluationReference:performanceEvaluationTypeDescriptor::string') }}     as performance_evaluation_type,
+        v:performanceEvaluationRatingReference:personId::string                                           as person_id,
+        v:performanceEvaluationRatingReference:schoolYear::int                                            as school_year,
+        {{ extract_descriptor('v:performanceEvaluationRatingReference:sourceSystemDescriptor::string') }} as source_system,
+        {{ extract_descriptor('v:performanceEvaluationRatingReference:termDescriptor::string') }}         as academic_term,
+        -- non-identity components
+        v:sectionReference:sectionIdentifier::string as section_id,
         -- descriptors
         {{ extract_descriptor('v:evaluationRatingStatusDescriptor::string') }} as evaluation_rating_status,
         {{ extract_descriptor('v:evaluationRatingLevelDescriptor::string') }}  as evaluation_rating_level,
@@ -23,7 +34,9 @@ renamed as (
         -- references
         v:evaluationReference                  as evaluation_reference,
         v:performanceEvaluationRatingReference as performance_evaluation_rating_reference,
-        v:sectionReference                     as section_reference
+        v:sectionReference                     as section_reference,
+        -- edfi extensions
+        v:_ext as v_ext
     from evaluation_ratings
 )
 select * from renamed