-
Notifications
You must be signed in to change notification settings - Fork 0
/
Copy pathprefect.yaml
104 lines (98 loc) · 3.83 KB
/
prefect.yaml
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
# Welcome to your prefect.yaml file! You can use this file for storing and managing
# configuration for deploying your flows. We recommend committing this file to source
# control along with your flow code.
# Generic metadata about this project
name: CCDI Hub Data Loader
prefect-version: 2.14.10
# build section allows you to manage and build docker images
build: null
# push section allows you to manage if and how this project is uploaded to remote locations
push: null
# pull section allows you to provide instructions for cloning this project in remote locations
pull:
- prefect.deployments.steps.git_clone:
repository: https://github.com/CBIIT/ccdi-model.git
branch: 2.0.0
- prefect.deployments.steps.git_clone:
id: clone-step
repository: https://github.com/CBIIT/CCDI-Hub-Prefect-Dataloader.git
branch: PORTAL-1870_testing
include_submodules: True
- prefect.projects.steps.pip_install_requirements:
requirements_file: requirements.txt
directory: "{{ clone-step.directory }}"
stream_output: False
# the deployments section allows you to provide configuration for deploying flows
deployments:
- name: ccdi-hub-data-loading-sandbox
version: null
tags: ["Production"]
description: null
schedule: {}
flow_name: null
entrypoint: prefect_data_loader.py:ccdi_hub_data_loader
parameters:
secret_name: "{{ prefect.variables.ccdi_dataloader_sandbox_secret_name }}"
metadata_folder: ""
runner: ""
model_tag: "2.0.0"
cheat_mode: False
dry_run: False
wipe_db: False
mode: "upsert"
split_transaction: True
work_pool:
name: ccdi-hub-sandbox-prefect-8gb
work_queue_name: default
job_variables:
env:
PREFECT_LOGGING_EXTRA_LOGGERS: "Data Loader,Loader,ICDC Schema,S3 Bucket,Bento Config,Copier,ESLoader,FileLoader,File Loader,Model Converter,Props,UUID_util,Bento_adapter,SQS,Visibility Extender,VisitCreator,VisitCreator,Props"
pull:
- prefect.deployments.steps.git_clone:
repository: https://github.com/CBIIT/ccdi-model.git
branch: 2.0.0
- prefect.deployments.steps.git_clone:
id: clone-step
repository: https://github.com/CBIIT/CCDI-Hub-Prefect-Dataloader.git
branch: main
include_submodules: True
- prefect.projects.steps.pip_install_requirements:
requirements_file: requirements.txt
directory: "{{ clone-step.directory }}"
stream_output: False
- name: TEST-ccdi-hub-data-loading-sandbox
version: null
tags: ["TEST"]
description: null
schedule: {}
flow_name: null
entrypoint: prefect_data_loader.py:ccdi_hub_data_loader
parameters:
secret_name: "{{ prefect.variables.ccdi_dataloader_sandbox_secret_name }}"
metadata_folder: ""
runner: ""
model_tag: "2.0.0"
cheat_mode: False
dry_run: False
wipe_db: False
mode: "upsert"
split_transaction: True
pull:
- prefect.deployments.steps.git_clone:
repository: https://github.com/CBIIT/ccdi-model.git
branch: 2.0.0
- prefect.deployments.steps.git_clone:
id: clone-step
repository: https://github.com/CBIIT/CCDI-Hub-Prefect-Dataloader.git
branch: PORTAL-1870_testing
include_submodules: True
- prefect.projects.steps.pip_install_requirements:
requirements_file: requirements.txt
directory: "{{ clone-step.directory }}"
stream_output: False
work_pool:
name: ccdi-hub-sandbox-prefect-8gb
work_queue_name: default
job_variables:
env:
PREFECT_LOGGING_EXTRA_LOGGERS: "Data Loader,Loader,ICDC Schema,S3 Bucket,Bento Config,Copier,ESLoader,FileLoader,File Loader,Model Converter,Props,UUID_util,Bento_adapter,SQS,Visibility Extender,VisitCreator,VisitCreator,Props"