diff --git a/data.yml b/data.yml index 1e83fb8..ba7f5ed 100644 --- a/data.yml +++ b/data.yml @@ -1,2 +1,2 @@ -defaultStorageMapping: null +defaultStorageMapping: CORE fileVersion: 3 diff --git a/locations.yml b/locations.yml index 4f44f07..921c4ae 100644 --- a/locations.yml +++ b/locations.yml @@ -1,3 +1,7 @@ -defaultStorageMapping: null +defaultStorageMapping: CORE fileVersion: 1 -locations: [] +locations: + - CORE + - EXTERNAL_TABLES + - RAW_STAGE + - TPCH_SF1 diff --git a/nodeTypes/DatavaultbyScalefreeHub-6/run.sql.j2 b/nodeTypes/DatavaultbyScalefreeHub-6/run.sql.j2 index 3eef6cf..c12b100 100644 --- a/nodeTypes/DatavaultbyScalefreeHub-6/run.sql.j2 +++ b/nodeTypes/DatavaultbyScalefreeHub-6/run.sql.j2 @@ -52,7 +52,10 @@ WITH incoming AS ( new_records AS ( SELECT - "SRC".* + {% for col in source.columns %} + "SRC"."{{ col.name }}" + {%- if not loop.last -%}, {% endif %} + {% endfor %} FROM incoming "SRC" WHERE NOT EXISTS @@ -65,7 +68,12 @@ new_records AS ( ) - SELECT * FROM new_records + SELECT + {% for col in source.columns %} + "SRC"."{{ col.name }}" + {%- if not loop.last -%}, {% endif %} + {% endfor %} + FROM new_records "SRC" {% endfor %} diff --git a/nodeTypes/DatavaultbyScalefreeLink-7/run.sql.j2 b/nodeTypes/DatavaultbyScalefreeLink-7/run.sql.j2 index c30bbdf..5f60e94 100644 --- a/nodeTypes/DatavaultbyScalefreeLink-7/run.sql.j2 +++ b/nodeTypes/DatavaultbyScalefreeLink-7/run.sql.j2 @@ -44,7 +44,10 @@ WITH incoming AS new_records AS ( SELECT - "SRC".* + {% for col in source.columns %} + "SRC"."{{ col.name }}" + {%- if not loop.last -%}, {% endif %} + {% endfor %} FROM incoming "SRC" WHERE NOT EXISTS ( SELECT @@ -56,7 +59,12 @@ new_records AS ( QUALIFY ROW_NUMBER() OVER (PARTITION BY "{{ link_hashkey }}" ORDER BY "{{ datavault4coalesce.config.ldts_alias }}" ) = 1 ) -SELECT * FROM new_records +SELECT +{% for col in source.columns %} + "SRC"."{{ col.name }}" + {%- if not loop.last -%}, {% endif %} +{% endfor %} +FROM new_records "SRC" {% endfor %} diff --git a/nodeTypes/DatavaultbyScalefreeStage-4/create.sql.j2 b/nodeTypes/DatavaultbyScalefreeStage-4/create.sql.j2 index 67da4f4..46a4549 100644 --- a/nodeTypes/DatavaultbyScalefreeStage-4/create.sql.j2 +++ b/nodeTypes/DatavaultbyScalefreeStage-4/create.sql.j2 @@ -39,7 +39,7 @@ {% for col in source.columns %} {#- Print DV hash calculations -#} {% if col.hashDetails %} - {{ datavault4coalesce__hash(columns=col.hashDetails.columns, is_hashdiff=col.is_hd,datatype=col.dataType, algo=col.hashDetails.algorithm, multi_active_key=multi_active_key, main_hashkey_col=main_hashkey_col.hk_col) }} AS "{{ col.name }}" + {{ datavault4coalesce__hash(columns=col.hashDetails.columns, is_hashdiff=col.is_hd,datatype=col.dataType, algo=datavault4coalesce.config.hash, multi_active_key=multi_active_key, main_hashkey_col=main_hashkey_col.hk_col) }} AS "{{ col.name }}" {#- Print other table columns -#} {% else %} {{ get_source_transform(col) }} AS "{{ col.name }}" diff --git a/nodeTypes/Dimension-Dimension/create.sql.j2 b/nodeTypes/Dimension-Dimension/create.sql.j2 index 180fbd1..59949c2 100644 --- a/nodeTypes/Dimension-Dimension/create.sql.j2 +++ b/nodeTypes/Dimension-Dimension/create.sql.j2 @@ -1,52 +1,52 @@ {% if node.materializationType == 'table' %} - {{ stage('Create Dimension Table') }} - - CREATE OR REPLACE TABLE {{ ref_no_link(node.location.name, node.name) }} - ( - {% for col in columns %} - "{{ col.name }}" {{ col.dataType }} - {% if col.isSurrogateKey %} - identity - {% endif %} - {%- if not col.nullable %} NOT NULL - {%- if col.defaultValue | length > 0 %} DEFAULT {{ col.defaultValue }}{% endif %} - {% endif %} - {%- if col.description | length > 0 %} COMMENT '{{ col.description | escape }}'{% endif %} - {%- if not loop.last -%}, {% endif %} - {% endfor %} - ) - {%- if node.description | length > 0 %} COMMENT = '{{ node.description | escape }}'{% endif %} + {{ stage('Create Dimension Table') }} + + CREATE OR REPLACE TABLE {{ ref_no_link(node.location.name, node.name) }} + ( + {% for col in columns %} + "{{ col.name }}" {{ col.dataType }} + {% if col.isSurrogateKey %} + identity + {% endif %} + {%- if not col.nullable %} NOT NULL + {%- if col.defaultValue | length > 0 %} DEFAULT {{ col.defaultValue }}{% endif %} + {% endif %} + {%- if col.description | length > 0 %} COMMENT '{{ col.description | escape }}'{% endif %} + {%- if not loop.last -%}, {% endif %} + {% endfor %} + ) + {%- if node.description | length > 0 %} COMMENT = '{{ node.description | escape }}'{% endif %} {% elif node.materializationType == 'view' %} - {{ stage('Create Dimension View') }} - - CREATE OR REPLACE VIEW {{ ref_no_link(node.location.name, node.name) }} - ( - {% for col in columns %} - "{{ col.name }}" - {%- if col.description | length > 0 %} COMMENT '{{ col.description | escape }}'{% endif %} - {%- if not loop.last -%},{% endif %} - {% endfor %} - ) - {%- if node.description | length > 0 %} COMMENT = '{{ node.description | escape }}'{% endif %} - AS - {% for source in sources %} - - {% if loop.first %}SELECT {% endif %} - - {% for col in source.columns %} - {% if col.isSurrogateKey or col.isSystemUpdateDate or col.isSystemCreateDate %} + {{ stage('Create Dimension View') }} + + CREATE OR REPLACE VIEW {{ ref_no_link(node.location.name, node.name) }} + ( + {% for col in columns %} + "{{ col.name }}" + {%- if col.description | length > 0 %} COMMENT '{{ col.description | escape }}'{% endif %} + {%- if not loop.last -%},{% endif %} + {% endfor %} + ) + {%- if node.description | length > 0 %} COMMENT = '{{ node.description | escape }}'{% endif %} + AS + {% for source in sources %} + + {% if loop.first %}SELECT {% endif %} + + {% for col in source.columns %} + {% if col.isSurrogateKey or col.isSystemUpdateDate or col.isSystemCreateDate %} NULL - {% else %} + {% else %} {{ get_source_transform(col) }} - {% endif %} - AS "{{ col.name }}" - {%- if not loop.last -%}, {% endif %} - {% endfor %} - {{ source.join }} - - {% if not loop.last %} UNION ALL {% endif %} - {% endfor %} + {% endif %} + AS "{{ col.name }}" + {%- if not loop.last -%}, {% endif %} + {% endfor %} + {{ source.join }} + + {% if not loop.last %} UNION ALL {% endif %} + {% endfor %} {% endif %} diff --git a/nodeTypes/Dimension-Dimension/run.sql.j2 b/nodeTypes/Dimension-Dimension/run.sql.j2 index 3444f79..bdf1825 100644 --- a/nodeTypes/Dimension-Dimension/run.sql.j2 +++ b/nodeTypes/Dimension-Dimension/run.sql.j2 @@ -10,11 +10,11 @@ {% if node.materializationType == 'table' %} - {% if config.preSQL %} - {{ stage('Pre-SQL') }} - {{ config.preSQL }} - {% endif %} - + {% if config.preSQL %} + {{ stage('Pre-SQL') }} + {{ config.preSQL }} + {% endif %} + {% if is_type_2 %} {% for source in sources %} @@ -118,57 +118,73 @@ ) {% endif %} {% endfor %} - UNION ALL - /* Rows Needing To Be Updated Due To Changes To Non-Type-2 source.columns - This case merges only when there are changes in non-type-2 column updates, but no changes in type-2 columns*/ - SELECT - {%- for col in source.columns if not col.isSurrogateKey %} - {% if col.isSystemVersion or col.isSystemCreateDate or col.isSystemStartDate or col.isSystemEndDate %} - "DIM"."{{ col.name }}" - {% elif col.isSystemCurrentFlag %} - 'Y' - {% else %} - {{ get_source_transform(col) }} - {% endif %} - AS "{{ col.name }}", - {% endfor -%} - 'UPDATE_NON_TYPE2_ROWS' AS "DML_OPERATION" - {{ source.join }} - INNER JOIN {{ ref_no_link(node.location.name, node.name) }} "DIM" ON - {% for col in source.columns if col.isBusinessKey -%} - {% if not loop.first %} - AND - {% endif %} - {{ get_source_transform(col) }} = "DIM"."{{ col.name }}" - {% endfor %} - WHERE "DIM"."{{ get_value_by_column_attribute("isSystemCurrentFlag") }}" = 'Y' - AND ( - {% for col in source.columns if (col.isChangeTracking) -%} - {% if not loop.first %} - AND - {% endif %} - {{ get_source_transform(col) }} = "DIM"."{{ col.name }}" - {% endfor %} ) - {% for col in source.columns if not ( col.isBusinessKey or - col.isChangeTracking or - col.isSurrogateKey or - col.isSystemVersion or - col.isSystemCurrentFlag or - col.isSystemStartDate or - col.isSystemEndDate or - col.isSystemUpdateDate or - col.isSystemCreateDate) -%} - {% if loop.first %} - AND ( - {% endif %} - {% if not loop.first %} - OR - {% endif %} - NVL( CAST({{ get_source_transform(col) }} as STRING), '**NULL**') <> NVL( CAST("DIM"."{{ col.name }}" as STRING), '**NULL**') - {% if loop.last %} - ) - {% endif %} - {% endfor %} + {# The if-block below avoids unnecessary updates when no type 2 column changes are present #} + {% if source.columns + | rejectattr('isSurrogateKey') + | rejectattr('isBusinessKey') + | rejectattr('isChangeTracking') + | rejectattr('isSystemVersion') + | rejectattr('isSystemCurrentFlag') + | rejectattr('isSystemStartDate') + | rejectattr('isSystemEndDate') + | rejectattr('isSystemCreateDate') + | rejectattr('isSystemUpdateDate') + | list | length == 0 + %} + {# Skip Section #} + {% else %} + UNION ALL + /* Rows Needing To Be Updated Due To Changes To Non-Type-2 columns + This case merges only when there are changes in non-type-2 column updates, but no changes in type-2 columns*/ + SELECT + {%- for col in source.columns if not col.isSurrogateKey %} + {% if col.isSystemVersion or col.isSystemCreateDate or col.isSystemStartDate or col.isSystemEndDate %} + "DIM"."{{ col.name }}" + {% elif col.isSystemCurrentFlag %} + 'Y' + {% else %} + {{ get_source_transform(col) }} + {% endif %} + AS "{{ col.name }}", + {% endfor -%} + 'UPDATE_NON_TYPE2_ROWS' AS "DML_OPERATION" + {{ source.join }} + INNER JOIN {{ ref_no_link(node.location.name, node.name) }} "DIM" ON + {% for col in source.columns if col.isBusinessKey -%} + {% if not loop.first %} + AND + {% endif %} + {{ get_source_transform(col) }} = "DIM"."{{ col.name }}" + {% endfor %} + WHERE "DIM"."{{ get_value_by_column_attribute("isSystemCurrentFlag") }}" = 'Y' + AND ( + {% for col in source.columns if (col.isChangeTracking) -%} + {% if not loop.first %} + AND + {% endif %} + {{ get_source_transform(col) }} = "DIM"."{{ col.name }}" + {% endfor %} ) + {% for col in source.columns if not ( col.isBusinessKey or + col.isChangeTracking or + col.isSurrogateKey or + col.isSystemVersion or + col.isSystemCurrentFlag or + col.isSystemStartDate or + col.isSystemEndDate or + col.isSystemUpdateDate or + col.isSystemCreateDate) -%} + {% if loop.first %} + AND ( + {% endif %} + {% if not loop.first %} + OR + {% endif %} + NVL( CAST({{ get_source_transform(col) }} as STRING), '**NULL**') <> NVL( CAST("DIM"."{{ col.name }}" as STRING), '**NULL**') + {% if loop.last %} + ) + {% endif %} + {% endfor %} + {% endif %} ) AS "SRC" ON {% for col in source.columns if col.isBusinessKey -%} @@ -208,9 +224,9 @@ SELECT {% for col in source.columns if not col.isSurrogateKey %} {% if col.isSystemVersion %} - 1 + 1 {% elif col.isSystemCurrentFlag %} - 'Y' + 'Y' {% else %} {{ get_source_transform(col) }} {% endif %} @@ -271,25 +287,25 @@ ) {% endfor %} {% endif %} - - {% if config.postSQL %} - {{ stage('Post-SQL') }} - {{ config.postSQL }} - {% endif %} + + {% if config.postSQL %} + {{ stage('Post-SQL') }} + {{ config.postSQL }} + {% endif %} {% endif %} {% if config.testsEnabled %} - {% for test in node.tests %} - {% if test.runOrder == 'After' %} - {{ test_stage(test.name, test.continueOnFailure) }} - {{ test.templateString }} + {% for test in node.tests %} + {% if test.runOrder == 'After' %} + {{ test_stage(test.name, test.continueOnFailure) }} + {{ test.templateString }} {% endif %} - {% endfor %} + {% endfor %} - {% for column in columns %} - {% for test in column.tests %} - {{ test_stage(column.name + ": " + test.name) }} - {{ test.templateString }} - {% endfor %} - {% endfor %} + {% for column in columns %} + {% for test in column.tests %} + {{ test_stage(column.name + ": " + test.name) }} + {{ test.templateString }} + {% endfor %} + {% endfor %} {% endif %} diff --git a/nodeTypes/PersistentStage-persistentStage/create.sql.j2 b/nodeTypes/PersistentStage-persistentStage/create.sql.j2 index eb6d359..9a06d3d 100644 --- a/nodeTypes/PersistentStage-persistentStage/create.sql.j2 +++ b/nodeTypes/PersistentStage-persistentStage/create.sql.j2 @@ -6,7 +6,7 @@ {% for col in columns %} "{{ col.name }}" {{ col.dataType }} {% if col.isSurrogateKey %} - identity + identity {% endif %} {%- if not col.nullable %} NOT NULL {%- if col.defaultValue | length > 0 %} DEFAULT {{ col.defaultValue }}{% endif %} @@ -33,8 +33,8 @@ AS {% for source in sources %} - {% if not loop.last %} UNION ALL {% endif %} - {% endfor %} + {% if not loop.last %} UNION ALL {% endif %} + {% endfor %} {% for col in source.columns %} {% if col.isSurrogateKey or col.isSystemUpdateDate or col.isSystemCreateDate %} diff --git a/nodeTypes/PersistentStage-persistentStage/run.sql.j2 b/nodeTypes/PersistentStage-persistentStage/run.sql.j2 index 62ac105..0e5f08b 100644 --- a/nodeTypes/PersistentStage-persistentStage/run.sql.j2 +++ b/nodeTypes/PersistentStage-persistentStage/run.sql.j2 @@ -10,11 +10,11 @@ {% endfor %} {% if node.materializationType == 'table' %} - {% if config.preSQL %} - {{ stage('Pre-SQL') }} - {{ config.preSQL }} - {% endif %} - + {% if config.preSQL %} + {{ stage('Pre-SQL') }} + {{ config.preSQL }} + {% endif %} + {% if has_business_key and is_type_2 %} {% for source in sources %} @@ -206,9 +206,9 @@ SELECT {% for col in source.columns if not col.isSurrogateKey %} {% if col.isSystemVersion %} - 1 + 1 {% elif col.isSystemCurrentFlag %} - 'Y' + 'Y' {% else %} {{ get_source_transform(col) }} {% endif %} @@ -296,26 +296,26 @@ {{ source.join }} {% endfor %} {% endif %} - - {% if config.postSQL %} - {{ stage('Post-SQL') }} - {{ config.postSQL }} - {% endif %} - + + {% if config.postSQL %} + {{ stage('Post-SQL') }} + {{ config.postSQL }} + {% endif %} + {% endif %} {% if config.testsEnabled %} - {% for test in node.tests %} - {% if test.runOrder == 'After' %} - {{ test_stage(test.name, test.continueOnFailure) }} - {{ test.templateString }} + {% for test in node.tests %} + {% if test.runOrder == 'After' %} + {{ test_stage(test.name, test.continueOnFailure) }} + {{ test.templateString }} {% endif %} - {% endfor %} + {% endfor %} - {% for column in columns %} - {% for test in column.tests %} - {{ test_stage(column.name + ": " + test.name) }} - {{ test.templateString }} - {% endfor %} - {% endfor %} + {% for column in columns %} + {% for test in column.tests %} + {{ test_stage(column.name + ": " + test.name) }} + {{ test.templateString }} + {% endfor %} + {% endfor %} {% endif %} \ No newline at end of file diff --git a/nodeTypes/Stage-Stage/create.sql.j2 b/nodeTypes/Stage-Stage/create.sql.j2 index a685cd9..5689005 100644 --- a/nodeTypes/Stage-Stage/create.sql.j2 +++ b/nodeTypes/Stage-Stage/create.sql.j2 @@ -1,22 +1,22 @@ {% if node.override.create.enabled %} - - {{ node.override.create.script }} + + {{ node.override.create.script }} {% elif node.materializationType == 'table' %} - {{ stage('Create Stage Table') }} - - CREATE OR REPLACE TABLE {{ ref_no_link(node.location.name, node.name) }} - ( - {% for col in columns %} - "{{ col.name }}" {{ col.dataType }} - {%- if not col.nullable %} NOT NULL - {%- if col.defaultValue | length > 0 %} DEFAULT {{ col.defaultValue }}{% endif %} - {% endif %} - {%- if col.description | length > 0 %} COMMENT '{{ col.description | escape }}'{% endif %} - {%- if not loop.last -%}, {% endif %} - {% endfor %} - ) - {%- if node.description | length > 0 %} COMMENT = '{{ node.description | escape }}'{% endif %} + {{ stage('Create Stage Table') }} + + CREATE OR REPLACE TABLE {{ ref_no_link(node.location.name, node.name) }} + ( + {% for col in columns %} + "{{ col.name }}" {{ col.dataType }} + {%- if not col.nullable %} NOT NULL + {%- if col.defaultValue | length > 0 %} DEFAULT {{ col.defaultValue }}{% endif %} + {% endif %} + {%- if col.description | length > 0 %} COMMENT '{{ col.description | escape }}'{% endif %} + {%- if not loop.last -%}, {% endif %} + {% endfor %} + ) + {%- if node.description | length > 0 %} COMMENT = '{{ node.description | escape }}'{% endif %} {% elif node.materializationType == 'view' %} diff --git a/nodeTypes/Stage-Stage/run.sql.j2 b/nodeTypes/Stage-Stage/run.sql.j2 index d357fb0..16ec527 100644 --- a/nodeTypes/Stage-Stage/run.sql.j2 +++ b/nodeTypes/Stage-Stage/run.sql.j2 @@ -7,79 +7,79 @@ {% endfor %} {% if node.materializationType == 'table' %} - {% if config.preSQL %} - {{ stage('Pre-SQL') }} - {{ config.preSQL }} - {% endif %} - - - - {% if config.truncateBefore %} - - {{ stage('Truncate Stage Table') }} - TRUNCATE IF EXISTS {{ ref_no_link(node.location.name, node.name) }} - - {% endif %} - - - {% if config.insertStrategy in ['UNION', 'UNION ALL'] %} - {{ stage( config.insertStrategy + ' Sources' | string ) }} - INSERT INTO {{ ref_no_link(node.location.name, node.name) }} - ( - {% for col in columns %} - "{{ col.name }}" - {%- if not loop.last -%},{% endif %} - {% endfor %} - ) - {% endif %} - - - {% for source in sources %} - - {% if config.insertStrategy == 'INSERT' %} - {{ stage('Insert ' + source.name | string ) }} - - INSERT INTO {{ ref_no_link(node.location.name, node.name) }} - ( - {% for col in source.columns %} - "{{ col.name }}" - {%- if not loop.last -%},{% endif %} - {% endfor %} - ) - {% endif %} - - SELECT - {% for col in source.columns %} + {% if config.preSQL %} + {{ stage('Pre-SQL') }} + {{ config.preSQL }} + {% endif %} + + + + {% if config.truncateBefore %} + + {{ stage('Truncate Stage Table') }} + TRUNCATE IF EXISTS {{ ref_no_link(node.location.name, node.name) }} + + {% endif %} + + + {% if config.insertStrategy in ['UNION', 'UNION ALL'] %} + {{ stage( config.insertStrategy + ' Sources' | string ) }} + INSERT INTO {{ ref_no_link(node.location.name, node.name) }} + ( + {% for col in columns %} + "{{ col.name }}" + {%- if not loop.last -%},{% endif %} + {% endfor %} + ) + {% endif %} + + + {% for source in sources %} + + {% if config.insertStrategy == 'INSERT' %} + {{ stage('Insert ' + source.name | string ) }} + + INSERT INTO {{ ref_no_link(node.location.name, node.name) }} + ( + {% for col in source.columns %} + "{{ col.name }}" + {%- if not loop.last -%},{% endif %} + {% endfor %} + ) + {% endif %} + + SELECT + {% for col in source.columns %} {{ get_source_transform(col) }} AS "{{ col.name }}" - {%- if not loop.last -%}, {% endif %} - {% endfor %} - - {{ source.join }} - - {% if config.insertStrategy in ['UNION', 'UNION ALL'] and not loop.last %} - {{config.insertStrategy}} - {% endif %} - - {% endfor %} - - {% if config.postSQL %} - {{ stage('Post-SQL') }} - {{ config.postSQL }} - {% endif %} + {%- if not loop.last -%}, {% endif %} + {% endfor %} + + {{ source.join }} + + {% if config.insertStrategy in ['UNION', 'UNION ALL'] and not loop.last %} + {{config.insertStrategy}} + {% endif %} + + {% endfor %} + + {% if config.postSQL %} + {{ stage('Post-SQL') }} + {{ config.postSQL }} + {% endif %} {% endif %} {% if config.testsEnabled %} - {% for test in node.tests %} - {% if test.runOrder == 'After' %} - {{ test_stage(test.name, test.continueOnFailure) }} - {{ test.templateString }} + {% for test in node.tests %} + {% if test.runOrder == 'After' %} + {{ test_stage(test.name, test.continueOnFailure) }} + {{ test.templateString }} {% endif %} - {% endfor %} + {% endfor %} - {% for column in columns %} - {% for test in column.tests %} - {{ test_stage(column.name + ": " + test.name) }} - {{ test.templateString }} - {% endfor %} - {% endfor %} + {% for column in columns %} + {% for test in column.tests %} + {{ test_stage(column.name + ": " + test.name) }} + {{ test.templateString }} + {% endfor %} + {% endfor %} {% endif %}