Merge pull request #24 from ScalefreeCOM/structure_upgrade_2

Upgraded the Project Structure to new multi-file structure

locations.yml

@@ -0,0 +1,3 @@
+defaultStorageMapping: null
+fileVersion: 1
+locations: []

nodeTypes/DatavaultbyScalefreeHub-6/create.sql.j2

@@ -0,0 +1,14 @@
+{{ stage('Create Hub Table') }}
+
+CREATE OR REPLACE TABLE {{ ref_no_link(node.location.name, node.name) }}
+(
+  {% for col in columns %}
+    "{{ col.name }}" {{ col.dataType }}
+    {%- if not col.nullable %} NOT NULL
+      {%- if col.defaultValue | length > 0 %} DEFAULT {{ col.defaultValue }}{% endif %}
+    {% endif %}
+    {%- if col.description | length > 0 %} COMMENT '{{ col.description }}'{% endif %}
+    {%- if not loop.last -%}, {% endif %}
+  {% endfor %}
+)
+{%- if node.description | length > 0 %} COMMENT = '{{ node.description }}'{% endif %}

nodeTypes/DatavaultbyScalefreeHub-6/definition.yml

@@ -0,0 +1,63 @@
+fileVersion: 1
+id: "6"
+isDisabled: false
+metadata:
+  defaultStorageLocation: null
+  error: null
+  nodeMetadataSpec: |-
+    capitalized: 'DATAVAULT BY SCALEFREE: HUB'
+    short: 'HUB'
+    tagColor: blue
+    plural: 'DATAVAULT BY SCALEFREE: HUBS'
+
+    config:
+    - groupName: Options
+      items:
+      - type: materializationSelector
+        isRequired: true
+        enableIf: 'true'
+        options: 
+        - table
+        default: table
+     
+      - displayName: Pre-SQL
+        attributeName: preSQL
+        type: textBox
+        isRequired: false
+
+      - displayName: Post-SQL
+        attributeName: postSQL
+        type: textBox
+        isRequired: false
+
+      - displayName: Enable Tests
+        attributeName: testsEnabled
+        type: toggleButton
+        default: true
+
+      - type: multisourceToggle
+
+    - groupName: Data Vault
+      items:
+
+      - displayName: Hub Hash Key column
+        type: columnSelector
+        attributeName: is_Hub_hk
+        isRequired: true
+        enableIf: 'true'
+        
+      - displayName: Hub LoadDateTimestamp Column
+        enableIf: 'false'
+        type: columnSelector
+        attributeName: is_Hub_ldts
+        isRequired: false
+
+      - displayName: Disable High-Water-Mark?
+        type: toggleButton
+        isRequired: false
+        enableIf: 'false'
+        attributeName: disable_hwm   
+        default: true
+        
+name: "Datavault by Scalefree: Hub"
+type: NodeType

nodeTypes/DatavaultbyScalefreeHub-6/run.sql.j2

@@ -0,0 +1,91 @@
+{% if config.preSQL %}
+  {{ stage('Pre-SQL') }}
+  {{ config.preSQL }}
+{% endif %}
+
+{% for test in node.tests if config.testsEnabled %}
+  {% if test.runOrder == 'Before' %}
+    {{ test_stage(test.name, test.continueOnFailure) }}
+    {{ test.templateString }}
+  {% endif %}
+{% endfor %}
+
+{% for source in sources %}
+
+  {{ stage('INSERT INTO Hub - ' ~ source.name) }}
+
+  {%- set hashkey_column = get_value_by_column_attribute("is_Hub_hk") -%}
+
+  {#-- enable when ColumnDropDownSelector is used again
+  {%- if 'is_Hub_hk' not in config.keys() -%}
+
+  {%- else -%}
+    
+    {%- set hashkey_column = config.is_Hub_hk.name -%}
+  
+  {%- endif -%}
+  #}
+
+  INSERT INTO {{ ref_no_link(node.location.name, node.name) }}
+
+WITH incoming AS (
+
+  SELECT DISTINCT
+  {% for col in source.columns %}
+    {{ get_source_transform(col) }} AS "{{ col.name }}"
+    {%- if not loop.last -%}, {% endif %}
+  {% endfor %}
+
+  {{ source.join }}
+
+    {% if not config.disable_hwm -%}
+    WHERE "{{ datavault4coalesce.config.ldts_alias }}" > (
+        SELECT 
+            COALESCE(MAX("{{ datavault4coalesce.config.ldts_alias }}"),  {{ datavault4coalesce__string_to_timestamp(datavault4coalesce.config.timestamp_format, datavault4coalesce.config.beginning_of_all_times) }})
+        FROM {{ ref_no_link(node.location.name, node.name) }}
+     WHERE "{{ datavault4coalesce.config.ldts_alias }}" != {{ datavault4coalesce__string_to_timestamp(datavault4coalesce.config.timestamp_format, datavault4coalesce.config.end_of_all_times) }}
+        )
+    {%- endif %}  
+
+), 
+
+new_records AS (
+
+  SELECT 
+  "SRC".* 
+  FROM incoming "SRC"
+  WHERE NOT EXISTS 
+
+  (SELECT 1 
+  FROM {{ ref_no_link(node.location.name, node.name) }} "TGT"
+  WHERE 
+    "SRC"."{{ hashkey_column }}" = "TGT"."{{ hashkey_column }}")    
+
+  QUALIFY ROW_NUMBER() OVER (PARTITION BY "{{ hashkey_column }}" ORDER BY "{{ datavault4coalesce.config.ldts_alias }}" ) = 1
+)
+
+
+  SELECT * FROM new_records
+
+{% endfor %}
+
+{% if config.postSQL %}
+  {{ stage('Post-SQL') }}
+  {{ config.postSQL }}
+{% endif %}
+
+{% if config.testsEnabled %}
+  {% for test in node.tests %}
+    {% if test.runOrder == 'After' %}
+      {{ test_stage(test.name, test.continueOnFailure) }}
+      {{ test.templateString }}
+        {% endif %}
+  {% endfor %}
+
+  {% for column in columns %}
+    {% for test in column.tests %}
+      {{ test_stage(column.name + ": " + test.name) }}
+      {{ test.templateString }}
+    {% endfor %}
+  {% endfor %}
+{% endif %}

nodeTypes/DatavaultbyScalefreeLink-7/create.sql.j2

@@ -0,0 +1,14 @@
+{{ stage('Create Link Table') }}
+
+CREATE OR REPLACE TABLE {{ ref_no_link(node.location.name, node.name) }}
+(
+    {% for col in columns %}
+        "{{ col.name }}" {{ col.dataType }}
+        {%- if not col.nullable %} NOT NULL
+            {%- if col.defaultValue | length > 0 %} DEFAULT {{ col.defaultValue }}{% endif %}
+        {% endif %}
+        {%- if col.description | length > 0 %} COMMENT '{{ col.description }}'{% endif %}
+        {%- if not loop.last -%}, {% endif %}
+    {% endfor %}
+)
+{%- if node.description | length > 0 %} COMMENT = '{{ node.description }}'{% endif %}

nodeTypes/DatavaultbyScalefreeLink-7/definition.yml

@@ -0,0 +1,54 @@
+fileVersion: 1
+id: "7"
+isDisabled: false
+metadata:
+  defaultStorageLocation: null
+  error: null
+  nodeMetadataSpec: |-
+    capitalized: 'DATAVAULT BY SCALEFREE: LINK'
+    short: 'LINK'
+    tagColor: 'Red'
+    plural: 'DATAVAULT BY SCALEFREE: LINKS'
+
+    config:
+    - groupName: Options
+      items:
+      - type: materializationSelector
+        isRequired: true
+        enableIf: 'false'
+        default: table
+        options:
+        - table
+
+      - displayName: Pre-SQL
+        attributeName: preSQL
+        type: textBox
+        isRequired: false
+
+      - displayName: Post-SQL
+        attributeName: postSQL
+        type: textBox
+        isRequired: false
+
+      - displayName: Enable Tests
+        attributeName: testsEnabled
+        type: toggleButton
+        default: true
+        
+      - type: multisourceToggle
+      
+    - groupName: Data Vault
+      items:
+      - displayName: Link Hashkey Column
+        type: columnSelector
+        attributeName: is_Link_hk
+        isRequired: true 
+
+      - displayName: Disable High-Water-Mark?
+        type: toggleButton
+        isRequired: false
+        enableIf: 'false'
+        attributeName: disable_hwm   
+        default: true    
+name: "Datavault by Scalefree: Link"
+type: NodeType

nodeTypes/DatavaultbyScalefreeLink-7/run.sql.j2

@@ -0,0 +1,82 @@
+{% if config.preSQL %}
+    {{ stage('Pre-SQL') }}
+    {{ config.preSQL }}
+
+{% endif %}
+
+{% for test in node.tests if config.testsEnabled %}
+  {% if test.runOrder == 'Before' %}
+    {{ test_stage(test.name, test.continueOnFailure) }}
+    {{ test.templateString }}
+  {% endif %}
+{% endfor %}
+
+{% for source in sources %}
+
+    {{ stage('INSERT INTO Link - ' ~ source.name) }}
+
+    {#--USE AGAIN WHEN COLUMNDROPDOWNSELECTOR IS USED%- set link_hashkey = config.is_Link_hk.name -%#}
+
+    {%- set link_hashkey = get_value_by_column_attribute("is_Link_hk") -%}
+
+
+    INSERT INTO {{ ref_no_link(node.location.name, node.name) }}
+
+WITH incoming AS 
+    (
+        SELECT DISTINCT
+        {% for col in source.columns %}
+            {{ get_source_transform(col) }} AS "{{ col.name }}"
+            {%- if not loop.last -%}, {% endif %}
+        {% endfor %}
+
+        {{ source.join }}
+
+
+    {% if not config.disable_hwm -%}
+    WHERE "{{ datavault4coalesce.config.ldts_alias }}" > (
+        SELECT 
+            COALESCE(MAX("{{ datavault4coalesce.config.ldts_alias }}"),  {{ datavault4coalesce__string_to_timestamp(datavault4coalesce.config.timestamp_format, datavault4coalesce.config.beginning_of_all_times) }})
+        FROM {{ ref_no_link(node.location.name, node.name) }}
+        )
+    {%- endif %}
+    ),
+
+new_records AS (
+    SELECT
+        "SRC".*
+    FROM incoming "SRC"
+    WHERE NOT EXISTS (
+        SELECT
+            1
+        FROM {{ ref_no_link(node.location.name, node.name) }} "TGT"
+        WHERE "SRC"."{{ link_hashkey }}" = "TGT"."{{ link_hashkey }}"
+    )
+
+    QUALIFY ROW_NUMBER() OVER (PARTITION BY "{{ link_hashkey }}" ORDER BY "{{ datavault4coalesce.config.ldts_alias }}" ) = 1
+)
+
+SELECT * FROM new_records
+
+{% endfor %}
+
+{% if config.postSQL %}
+    {{ stage('Post-SQL') }}
+    {{ config.postSQL }}    
+{% endif %}
+
+{% if config.testsEnabled %}
+  {% for test in node.tests %}
+    {% if test.runOrder == 'After' %}
+      {{ test_stage(test.name, test.continueOnFailure) }}
+      {{ test.templateString }}
+        {% endif %}
+  {% endfor %}
+
+  {% for column in columns %}
+    {% for test in column.tests %}
+      {{ test_stage(column.name + ": " + test.name) }}
+      {{ test.templateString }}
+    {% endfor %}
+  {% endfor %}
+{% endif %}

nodeTypes/DatavaultbyScalefreeMulti-activeSatelliteV0-18/create.sql.j2

@@ -0,0 +1,14 @@
+{{ stage('Create Satellite Table') }}
+
+CREATE OR REPLACE TABLE {{ ref_no_link(node.location.name, node.name) }}
+(
+  {% for col in columns %}
+    "{{ col.name }}" {{ col.dataType }}
+    {%- if not col.nullable %} NOT NULL
+      {%- if col.defaultValue | length > 0 %} DEFAULT {{ col.defaultValue }}{% endif %}
+    {% endif %}
+    {%- if col.description | length > 0 %} COMMENT '{{ col.description }}'{% endif %}
+    {%- if not loop.last -%}, {% endif %}
+  {% endfor %}
+)
+{%- if node.description | length > 0 %} COMMENT = '{{ node.description }}'{% endif %}

nodeTypes/DatavaultbyScalefreeMulti-activeSatelliteV0-18/definition.yml

@@ -0,0 +1,43 @@
+fileVersion: 1
+id: "18"
+isDisabled: false
+metadata:
+  defaultStorageLocation: null
+  error: null
+  nodeMetadataSpec: |-
+    capitalized: 'DATAVAULT BY SCALEFREE: MULTI-ACTIVE SATELLITE V0'
+    short: 'SATMV0'
+    plural: 'DATAVAULT BY SCALEFREE: MULTI-ACTIVE SATELLITES V0'
+    tagColor: '#dfdf1f'
+
+    config:
+    - groupName: Data Vault
+      items:
+      - displayName: Hashkey Column
+        type: columnSelector
+        attributeName: is_hk
+        isRequired: true
+      
+      - displayName: Multi-Active Key Column
+        type: columnSelector
+        attributeName: is_ma_key
+        isRequired: true
+
+      - displayName: Hashdiff Column
+        type: columnSelector
+        attributeName: is_hd
+        isRequired: true
+
+    - groupName: Pre/Post SQL
+      items:
+      - displayName: Pre-SQL
+        attributeName: preSQL
+        type: textBox
+        isRequired: false
+
+      - displayName: Post-SQL
+        attributeName: postSQL
+        type: textBox
+        isRequired: false
+name: "Datavault by Scalefree: Multi-active Satellite V0"
+type: NodeType