-
Notifications
You must be signed in to change notification settings - Fork 0
/
Copy pathedit-template.yaml
496 lines (486 loc) · 18.4 KB
/
edit-template.yaml
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
apiVersion: witboost.com/v1
kind: EditTemplate
metadata:
name: edit-azure-outputport-adlsgen2-template.1
title: ADLS Gen2 Output Port Edit Template
description: Updates an Output Port based on a directory located on an ADLS Gen2 Storage Account
mesh:
icon: https://cdn-images-1.medium.com/v2/resize:fit:914/1*iYJgfYgxDWBcfWjl1twsag.png
tags:
- azure
- adlsgen2
- outputport
- edit
spec:
owner: agilelab
useCaseTemplateId:
- urn:dmb:utm:azure-outputport-adlsgen2-template:0.0.0
parameters:
- title: Update Output Port
description: |
This wizard will allow you to edit the metadata for a previously created Output Port.
Most metadata is editable but some (like the name, domain, data product etc) are not; their fields will be read-only.
Some of the files in the component's repository cannot be updated by this wizard, namely:
- **Documentation**: `README.md`, `docs/index.md`, etc
- **CI/CD**: `.gitlab-ci-yaml`
You need to take care of updating these yourself, as given their nature it's hard to do automatically and they are often edited manually after the component's creation.
- title: Update component metadata
required:
- description
- dependencies
properties:
name:
title: Name
type: string
description: Required name used for display purposes
ui:disabled: true
description:
title: Description
type: string
description: Help others understand what this component is for
domain:
title: Domain
type: string
description: Domain of the Data Product this component belongs to
ui:field: EntityPicker
ui:options:
allowArbitraryValues: false
allowedKinds:
- Domain
ui:disabled: true
dataproduct:
title: Data Product
type: string
description: Data Product this Workload belongs to
ui:field: EntityPicker
ui:filter:
- fieldName: domain
entityPath: spec.domain
ui:options:
allowArbitraryValues: false
allowedKinds:
- System
ui:disabled: true
identifier:
title: Identifier
type: string
description: A unique identifier for the entity inside the domain. It will not be editable after creation and is expected to be a string that is sequences of [a-zA-Z] separated by any of [-_]
ui:field: ComponentIdentifierPicker
ui:options:
allowArbitraryValues: false
developmentGroup:
title: Development Group
type: string
description: Data Product development group
ui:field: EntitySelectionPicker
ui:fieldName: dataproduct
ui:property: spec.owner
ui:options:
allowArbitraryValues: false
dependencies:
type: object
ui:options:
displayTitle: false
required:
- dependsOn
- containerList
properties:
dependsOn:
title: Storage component dependency
description: ADLS Gen2 Output Ports must depend on an ADLS Gen2 Storage Area in order to extract the storage account and container information
type: string
ui:field: EntityRelationsPicker
ui:fieldName: dataproduct
relation: hasPart
ui:filters:
- key: spec.mesh.useCaseTemplateId
value: "urn:dmb:utm:azure-storage-adlsgen2-template:0.0.0"
containerList:
ui:widget: hidden
title: Container list
ui:field: EntitySelectionPicker
ui:fieldName: dependencies.dependsOn
ui:property: spec.mesh.specific.containers
tags:
title: Tags
type: array
description: Tags for the component
default: [ ]
items:
type: string
- title: Terms & Conditions and SLA
description: Terms & Conditions and Service Level Agreement information for this component.
required:
- termsAndConditions
- intervalOfChange
- timeliness
- upTime
properties:
termsAndConditions:
title: Terms and Conditions
type: string
description: Required. Terms and Conditions that apply to the data.
default: Can be used for production purposes.
intervalOfChange:
title: Interval of change
type: string
description: Required. How often the data is refreshed.
default: 2BD
timeliness:
title: Timeliness
type: string
description: Required. Timeliness of the data.
default: 2BD
upTime:
title: Uptime
type: string
description: Required. Uptime of the Output Port.
default: 99.9%
- title: Data Sharing Agreement
description: Update the Data Sharing Agreement clauses for this component.
required:
- purpose
- billing
- security
- intendedUsage
- limitations
- lifeCycle
- confidentiality
properties:
purpose:
title: Purpose
type: string
description: Required. Purpose of the data.
default: Foundational data for downstream use cases.
billing:
title: Billing
type: string
description: Required. Billing applied to data usage.
default: None.
security:
title: Security
type: string
description: Required. Security policies that apply to the data.
default: Platform standard security policies.
intendedUsage:
title: Intended Usage
type: string
description: Required. Intended usage for the data.
default: Any downstream use cases.
limitations:
title: Limitations
type: string
description: Required. Limitations of the data.
default: Needs joining with other datasets (eg customer data) for most analytical use cases.
lifeCycle:
title: Lifecycle
type: string
description: Required. Lifecycle of the data.
default: Data loaded every two days and typically never deleted.
confidentiality:
title: Confidentiality
type: string
description: Required. Confidentiality of the data.
default: None.
- title: Data contract schema
description: Definition of the columns of the table. Primitive data types supported.
properties:
schemaDefinition:
type: object
ui:options:
displayTitle: false
properties:
schemaColumns:
title: Column Definitions
type: array
ui:ArrayFieldTemplate: ArrayTableTemplate
items:
type: object
ui:ObjectFieldTemplate: TableRowTemplate
required:
- name
- dataType
properties:
name:
type: string
title: Name
description:
type: string
title: Description
dataType:
type: string
default: INT
title: Data Type
enum:
- TINYINT
- SMALLINT
- INT
- BIGINT
- DOUBLE
- DECIMAL
- TIMESTAMP
- DATE
- STRING
- TEXT
- CHAR
- VARCHAR
- BOOLEAN
- ARRAY
constraint:
type: string
title: Constraint
default: (no constraint)
enum:
- PRIMARY_KEY
- NOT_NULL
- UNIQUE
- (no constraint)
allOf:
- if:
properties:
dataType:
const: ARRAY
then:
required:
- arrayDataType
properties:
arrayDataType:
title: Array Data Type
description: Data type of the array elements
type: string
default: INT
enum:
- TINYINT
- SMALLINT
- INT
- BIGINT
- DOUBLE
- DECIMAL
- TIMESTAMP
- DATE
- STRING
- TEXT
- CHAR
- VARCHAR
- BOOLEAN
- ARRAY
allOf:
- if:
properties:
arrayDataType:
oneOf:
- const: TEXT
- const: VARCHAR
- const: CHAR
then:
properties:
dataLength:
title: Column Length
type: integer
description: Maximum length of column row
default: 65535
required:
- dataLength
- if:
properties:
arrayDataType:
oneOf:
- const: DECIMAL
then:
properties:
precision:
title: Precision
type: integer
description: Precision of the numeric data type
minimum: 1
maximum: 38
default: 38
scale:
title: Scale
type: integer
description: Scale of the numeric data type; cannot be greater than precision minus 1
minimum: 0
maximum: 37
default: 0
required:
- precision
- scale
- if:
properties:
dataType:
oneOf:
- const: TEXT
- const: VARCHAR
- const: CHAR
then:
properties:
dataLength:
title: Column Length
type: integer
description: Maximum length of column row
default: 65535
required:
- dataLength
- if:
properties:
dataType:
oneOf:
- const: DECIMAL
then:
properties:
precision:
title: Precision
type: integer
description: Precision of the numeric data type
minimum: 1
maximum: 38
default: 38
scale:
title: Scale
type: integer
description: Scale of the numeric data type; cannot be greater than precision minus 1
minimum: 0
maximum: 37
default: 0
required:
- precision
- scale
- if: true
then:
properties:
businessTerms:
title: Business Terms
type: array
description: Multiple selection for fixed tags
uniqueItems: true
ui:style:
minWidth: 400
items:
type: string
enum:
- Raw
- Derived
- Metadata
- Identifier
- Dimension
- Fact
- title: Provide ADLS Gen2 directory information
description: |-
This template will expose a folder inside an existing ADLS Gen2 Storage Account, creating the folder if not existent.
The **Storage Account** name will be retrieved from the Storage Area dependency output.
You can leave these values empty for the meantime, but in order for the deployment to be successful, they must be filled out
required:
- containerName
- folder
properties:
containerName:
title: Container name
type: object
description: Name of the container. This container must exist when deploying this component
ui:field: DescriptorPicker
sourceType: field
source: dependencies
optionsAt: containerList
folder:
title: Folder specification
type: object
required:
- autogeneratePath
properties:
autogeneratePath:
title: Generate folder path automatically
type: boolean
default: true
dependencies:
autogeneratePath:
oneOf:
- properties:
autogeneratePath:
const: true
prefixPath:
title: Prefix path
type: string
description: "Optional: Specify a prefix path to prepended the autogenerated path."
pattern: /$
ui:help: If provided, it must end with a "/" character
- required:
- folderPath
properties:
autogeneratePath:
const: false
folderPath:
title: Folder path
type: string
description: Path inside the container for the directory to be created and exposed.
ui:field: RegexPicker
pattern: ^[^/]+(?:/[^/]+)*$
ui:help: The path must not start or end with a "/" character
ui:options:
allowArbitraryValues: true
format:
type: object
ui:options:
displayTitle: false
ui:ObjectFieldTemplate: HorizontalTemplate
required:
- fileFormat
properties:
fileFormat:
title: Files format
type: string
description: Select the file format of data stored inside the Output Port directory. This will be shown to the end-user
enum:
- CSV
- Parquet
- DeltaLake
- Iceberg
- Other
allOf:
- if:
properties:
fileFormat:
const: Other
required: [ fileFormat ]
then:
required:
- otherFileFormat
properties:
otherFileFormat:
title: Other (please specify)
type: string
description: Specify the file format of the files contained in the directory
steps:
- id: template
name: Fetch Skeleton + Template
action: fetch:template
input:
url: ./edit-skeleton
targetPath: '.'
values:
name: '${{ parameters.name }}'
description: '${{ parameters.description }}'
domain: '${{ parameters.domain }}'
dataproduct: '${{ parameters.dataproduct }}'
identifier: '${{ parameters.identifier }}'
developmentGroup: '${{ parameters.developmentGroup }}'
dependsOn: '${{ parameters.dependencies.dependsOn }}'
processDescription: '${{ parameters.processDescription }}'
intervalOfChange: '${{ parameters.intervalOfChange }}'
timeliness: '${{ parameters.timeliness }}'
upTime: '${{ parameters.upTime }}'
termsAndConditions: '${{ parameters.termsAndConditions }}'
endpoint: '${{ parameters.endpoint }}'
schemaColumns: '${{ parameters.schemaDefinition.schemaColumns }}'
purpose: '${{ parameters.purpose }}'
billing: '${{ parameters.billing }}'
security: '${{ parameters.security }}'
intendedUsage: '${{ parameters.intendedUsage }}'
limitations: '${{ parameters.limitations }}'
lifeCycle: '${{ parameters.lifeCycle }}'
confidentiality: '${{ parameters.confidentiality }}'
containerName: '${{ parameters.containerName.value }}'
autogeneratePath: '${{ parameters.folder.autogeneratePath }}'
prefixPath: '${{ parameters.folder.prefixPath }}'
folderPath: '${{ parameters.folder.folderPath }}'
fileFormat: '${{ parameters.format.fileFormat }}'
otherFileFormat: '${{ parameters.format.otherFileFormat }}'