Skip to content

Commit

Permalink
Merge pull request #27 from ScalefreeCOM/dev_tkirschke
Browse files Browse the repository at this point in the history 
Fixed Column selection for Hubs and Links - Hash Algorithm Selection Fixed
  • Loading branch information
@tkirschke
tkirschke authored Nov 5, 2024
2 parents 40a0505 + e2035bd commit bccf44c
Show file tree
Hide file tree
Showing 11 changed files with 274 additions and 238 deletions.
2 changes: 1 addition & 1 deletion data.yml
View file
Original file line number Diff line number Diff line change
@@ -1,2 +1,2 @@
defaultStorageMapping: null
defaultStorageMapping: CORE
fileVersion: 3
8 changes: 6 additions & 2 deletions locations.yml
View file
Original file line number Diff line number Diff line change
@@ -1,3 +1,7 @@
defaultStorageMapping: null
defaultStorageMapping: CORE
fileVersion: 1
locations: []
locations:
- CORE
- EXTERNAL_TABLES
- RAW_STAGE
- TPCH_SF1
12 changes: 10 additions & 2 deletions nodeTypes/DatavaultbyScalefreeHub-6/run.sql.j2
View file
Original file line number Diff line number Diff line change
Expand Up @@ -52,7 +52,10 @@ WITH incoming AS (
new_records AS (

SELECT
"SRC".*
{% for col in source.columns %}
"SRC"."{{ col.name }}"
{%- if not loop.last -%}, {% endif %}
{% endfor %}
FROM incoming "SRC"
WHERE NOT EXISTS

Expand All @@ -65,7 +68,12 @@ new_records AS (
)


SELECT * FROM new_records
SELECT
{% for col in source.columns %}
"SRC"."{{ col.name }}"
{%- if not loop.last -%}, {% endif %}
{% endfor %}
FROM new_records "SRC"

{% endfor %}

Expand Down
12 changes: 10 additions & 2 deletions nodeTypes/DatavaultbyScalefreeLink-7/run.sql.j2
View file
Original file line number Diff line number Diff line change
Expand Up @@ -44,7 +44,10 @@ WITH incoming AS

new_records AS (
SELECT
"SRC".*
{% for col in source.columns %}
"SRC"."{{ col.name }}"
{%- if not loop.last -%}, {% endif %}
{% endfor %}
FROM incoming "SRC"
WHERE NOT EXISTS (
SELECT
Expand All @@ -56,7 +59,12 @@ new_records AS (
QUALIFY ROW_NUMBER() OVER (PARTITION BY "{{ link_hashkey }}" ORDER BY "{{ datavault4coalesce.config.ldts_alias }}" ) = 1
)

SELECT * FROM new_records
SELECT
{% for col in source.columns %}
"SRC"."{{ col.name }}"
{%- if not loop.last -%}, {% endif %}
{% endfor %}
FROM new_records "SRC"

{% endfor %}

Expand Down
2 changes: 1 addition & 1 deletion nodeTypes/DatavaultbyScalefreeStage-4/create.sql.j2
View file
Original file line number Diff line number Diff line change
Expand Up @@ -39,7 +39,7 @@
{% for col in source.columns %}
{#- Print DV hash calculations -#}
{% if col.hashDetails %}
{{ datavault4coalesce__hash(columns=col.hashDetails.columns, is_hashdiff=col.is_hd,datatype=col.dataType, algo=col.hashDetails.algorithm, multi_active_key=multi_active_key, main_hashkey_col=main_hashkey_col.hk_col) }} AS "{{ col.name }}"
{{ datavault4coalesce__hash(columns=col.hashDetails.columns, is_hashdiff=col.is_hd,datatype=col.dataType, algo=datavault4coalesce.config.hash, multi_active_key=multi_active_key, main_hashkey_col=main_hashkey_col.hk_col) }} AS "{{ col.name }}"
{#- Print other table columns -#}
{% else %}
{{ get_source_transform(col) }} AS "{{ col.name }}"
Expand Down
88 changes: 44 additions & 44 deletions nodeTypes/Dimension-Dimension/create.sql.j2
View file
Original file line number Diff line number Diff line change
@@ -1,52 +1,52 @@
{% if node.materializationType == 'table' %}
{{ stage('Create Dimension Table') }}

CREATE OR REPLACE TABLE {{ ref_no_link(node.location.name, node.name) }}
(
{% for col in columns %}
"{{ col.name }}" {{ col.dataType }}
{% if col.isSurrogateKey %}
identity
{% endif %}
{%- if not col.nullable %} NOT NULL
{%- if col.defaultValue | length > 0 %} DEFAULT {{ col.defaultValue }}{% endif %}
{% endif %}
{%- if col.description | length > 0 %} COMMENT '{{ col.description | escape }}'{% endif %}
{%- if not loop.last -%}, {% endif %}
{% endfor %}
)
{%- if node.description | length > 0 %} COMMENT = '{{ node.description | escape }}'{% endif %}
{{ stage('Create Dimension Table') }}

CREATE OR REPLACE TABLE {{ ref_no_link(node.location.name, node.name) }}
(
{% for col in columns %}
"{{ col.name }}" {{ col.dataType }}
{% if col.isSurrogateKey %}
identity
{% endif %}
{%- if not col.nullable %} NOT NULL
{%- if col.defaultValue | length > 0 %} DEFAULT {{ col.defaultValue }}{% endif %}
{% endif %}
{%- if col.description | length > 0 %} COMMENT '{{ col.description | escape }}'{% endif %}
{%- if not loop.last -%}, {% endif %}
{% endfor %}
)
{%- if node.description | length > 0 %} COMMENT = '{{ node.description | escape }}'{% endif %}


{% elif node.materializationType == 'view' %}
{{ stage('Create Dimension View') }}

CREATE OR REPLACE VIEW {{ ref_no_link(node.location.name, node.name) }}
(
{% for col in columns %}
"{{ col.name }}"
{%- if col.description | length > 0 %} COMMENT '{{ col.description | escape }}'{% endif %}
{%- if not loop.last -%},{% endif %}
{% endfor %}
)
{%- if node.description | length > 0 %} COMMENT = '{{ node.description | escape }}'{% endif %}
AS
{% for source in sources %}

{% if loop.first %}SELECT {% endif %}

{% for col in source.columns %}
{% if col.isSurrogateKey or col.isSystemUpdateDate or col.isSystemCreateDate %}
{{ stage('Create Dimension View') }}

CREATE OR REPLACE VIEW {{ ref_no_link(node.location.name, node.name) }}
(
{% for col in columns %}
"{{ col.name }}"
{%- if col.description | length > 0 %} COMMENT '{{ col.description | escape }}'{% endif %}
{%- if not loop.last -%},{% endif %}
{% endfor %}
)
{%- if node.description | length > 0 %} COMMENT = '{{ node.description | escape }}'{% endif %}
AS
{% for source in sources %}

{% if loop.first %}SELECT {% endif %}

{% for col in source.columns %}
{% if col.isSurrogateKey or col.isSystemUpdateDate or col.isSystemCreateDate %}
NULL
{% else %}
{% else %}
{{ get_source_transform(col) }}
{% endif %}
AS "{{ col.name }}"
{%- if not loop.last -%}, {% endif %}
{% endfor %}
{{ source.join }}

{% if not loop.last %} UNION ALL {% endif %}
{% endfor %}
{% endif %}
AS "{{ col.name }}"
{%- if not loop.last -%}, {% endif %}
{% endfor %}
{{ source.join }}

{% if not loop.last %} UNION ALL {% endif %}
{% endfor %}

{% endif %}
164 changes: 90 additions & 74 deletions nodeTypes/Dimension-Dimension/run.sql.j2
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,11 +10,11 @@

{% if node.materializationType == 'table' %}

{% if config.preSQL %}
{{ stage('Pre-SQL') }}
{{ config.preSQL }}
{% endif %}
{% if config.preSQL %}
{{ stage('Pre-SQL') }}
{{ config.preSQL }}
{% endif %}
{% if is_type_2 %}

{% for source in sources %}
Expand Down Expand Up @@ -118,57 +118,73 @@
)
{% endif %}
{% endfor %}
UNION ALL
/* Rows Needing To Be Updated Due To Changes To Non-Type-2 source.columns
This case merges only when there are changes in non-type-2 column updates, but no changes in type-2 columns*/
SELECT
{%- for col in source.columns if not col.isSurrogateKey %}
{% if col.isSystemVersion or col.isSystemCreateDate or col.isSystemStartDate or col.isSystemEndDate %}
"DIM"."{{ col.name }}"
{% elif col.isSystemCurrentFlag %}
'Y'
{% else %}
{{ get_source_transform(col) }}
{% endif %}
AS "{{ col.name }}",
{% endfor -%}
'UPDATE_NON_TYPE2_ROWS' AS "DML_OPERATION"
{{ source.join }}
INNER JOIN {{ ref_no_link(node.location.name, node.name) }} "DIM" ON
{% for col in source.columns if col.isBusinessKey -%}
{% if not loop.first %}
AND
{% endif %}
{{ get_source_transform(col) }} = "DIM"."{{ col.name }}"
{% endfor %}
WHERE "DIM"."{{ get_value_by_column_attribute("isSystemCurrentFlag") }}" = 'Y'
AND (
{% for col in source.columns if (col.isChangeTracking) -%}
{% if not loop.first %}
AND
{% endif %}
{{ get_source_transform(col) }} = "DIM"."{{ col.name }}"
{% endfor %} )
{% for col in source.columns if not ( col.isBusinessKey or
col.isChangeTracking or
col.isSurrogateKey or
col.isSystemVersion or
col.isSystemCurrentFlag or
col.isSystemStartDate or
col.isSystemEndDate or
col.isSystemUpdateDate or
col.isSystemCreateDate) -%}
{% if loop.first %}
AND (
{% endif %}
{% if not loop.first %}
OR
{% endif %}
NVL( CAST({{ get_source_transform(col) }} as STRING), '**NULL**') <> NVL( CAST("DIM"."{{ col.name }}" as STRING), '**NULL**')
{% if loop.last %}
)
{% endif %}
{% endfor %}
{# The if-block below avoids unnecessary updates when no type 2 column changes are present #}
{% if source.columns
| rejectattr('isSurrogateKey')
| rejectattr('isBusinessKey')
| rejectattr('isChangeTracking')
| rejectattr('isSystemVersion')
| rejectattr('isSystemCurrentFlag')
| rejectattr('isSystemStartDate')
| rejectattr('isSystemEndDate')
| rejectattr('isSystemCreateDate')
| rejectattr('isSystemUpdateDate')
| list | length == 0
%}
{# Skip Section #}
{% else %}
UNION ALL
/* Rows Needing To Be Updated Due To Changes To Non-Type-2 columns
This case merges only when there are changes in non-type-2 column updates, but no changes in type-2 columns*/
SELECT
{%- for col in source.columns if not col.isSurrogateKey %}
{% if col.isSystemVersion or col.isSystemCreateDate or col.isSystemStartDate or col.isSystemEndDate %}
"DIM"."{{ col.name }}"
{% elif col.isSystemCurrentFlag %}
'Y'
{% else %}
{{ get_source_transform(col) }}
{% endif %}
AS "{{ col.name }}",
{% endfor -%}
'UPDATE_NON_TYPE2_ROWS' AS "DML_OPERATION"
{{ source.join }}
INNER JOIN {{ ref_no_link(node.location.name, node.name) }} "DIM" ON
{% for col in source.columns if col.isBusinessKey -%}
{% if not loop.first %}
AND
{% endif %}
{{ get_source_transform(col) }} = "DIM"."{{ col.name }}"
{% endfor %}
WHERE "DIM"."{{ get_value_by_column_attribute("isSystemCurrentFlag") }}" = 'Y'
AND (
{% for col in source.columns if (col.isChangeTracking) -%}
{% if not loop.first %}
AND
{% endif %}
{{ get_source_transform(col) }} = "DIM"."{{ col.name }}"
{% endfor %} )
{% for col in source.columns if not ( col.isBusinessKey or
col.isChangeTracking or
col.isSurrogateKey or
col.isSystemVersion or
col.isSystemCurrentFlag or
col.isSystemStartDate or
col.isSystemEndDate or
col.isSystemUpdateDate or
col.isSystemCreateDate) -%}
{% if loop.first %}
AND (
{% endif %}
{% if not loop.first %}
OR
{% endif %}
NVL( CAST({{ get_source_transform(col) }} as STRING), '**NULL**') <> NVL( CAST("DIM"."{{ col.name }}" as STRING), '**NULL**')
{% if loop.last %}
)
{% endif %}
{% endfor %}
{% endif %}
) AS "SRC"
ON
{% for col in source.columns if col.isBusinessKey -%}
Expand Down Expand Up @@ -208,9 +224,9 @@
SELECT
{% for col in source.columns if not col.isSurrogateKey %}
{% if col.isSystemVersion %}
1
1
{% elif col.isSystemCurrentFlag %}
'Y'
'Y'
{% else %}
{{ get_source_transform(col) }}
{% endif %}
Expand Down Expand Up @@ -271,25 +287,25 @@
)
{% endfor %}
{% endif %}
{% if config.postSQL %}
{{ stage('Post-SQL') }}
{{ config.postSQL }}
{% endif %}
{% if config.postSQL %}
{{ stage('Post-SQL') }}
{{ config.postSQL }}
{% endif %}
{% endif %}

{% if config.testsEnabled %}
{% for test in node.tests %}
{% if test.runOrder == 'After' %}
{{ test_stage(test.name, test.continueOnFailure) }}
{{ test.templateString }}
{% for test in node.tests %}
{% if test.runOrder == 'After' %}
{{ test_stage(test.name, test.continueOnFailure) }}
{{ test.templateString }}
{% endif %}
{% endfor %}
{% endfor %}

{% for column in columns %}
{% for test in column.tests %}
{{ test_stage(column.name + ": " + test.name) }}
{{ test.templateString }}
{% endfor %}
{% endfor %}
{% for column in columns %}
{% for test in column.tests %}
{{ test_stage(column.name + ": " + test.name) }}
{{ test.templateString }}
{% endfor %}
{% endfor %}
{% endif %}
6 changes: 3 additions & 3 deletions nodeTypes/PersistentStage-persistentStage/create.sql.j2
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@
{% for col in columns %}
"{{ col.name }}" {{ col.dataType }}
{% if col.isSurrogateKey %}
identity
identity
{% endif %}
{%- if not col.nullable %} NOT NULL
{%- if col.defaultValue | length > 0 %} DEFAULT {{ col.defaultValue }}{% endif %}
Expand All @@ -33,8 +33,8 @@
AS
{% for source in sources %}

{% if not loop.last %} UNION ALL {% endif %}
{% endfor %}
{% if not loop.last %} UNION ALL {% endif %}
{% endfor %}

{% for col in source.columns %}
{% if col.isSurrogateKey or col.isSystemUpdateDate or col.isSystemCreateDate %}
Expand Down
Loading

0 comments on commit bccf44c

Please sign in to comment.