-
Notifications
You must be signed in to change notification settings - Fork 9
/
Copy pathgalaxy_elastic_cluster_elixir_it.yaml
310 lines (291 loc) · 10.6 KB
/
galaxy_elastic_cluster_elixir_it.yaml
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
tosca_definitions_version: tosca_simple_yaml_1_0
imports:
- indigo_custom_types: https://raw.githubusercontent.com/indigo-dc/tosca-types/master/custom_types.yaml
description: >
Deploy Galaxy from scratch with elastic cluster support (SLOW).
The basic configuration includes CentOS 7, SLURM, the selected Galaxy flavour, companion software and reference data.
Configure, click on the "Submit" button and wait for the confirmation e-mail(s) with instructions on how to provide your passphrase (if encryption is enabled) and log in to your new Galaxy instance.
If after some hours you do not receive any e-mail please be sure to check your SPAM BOX.
topology_template:
inputs:
wn_num_max:
type: integer
description: Maximum number of WNs in the elastic cluster
default: 1
required: yes
wn_num_min:
type: integer
description: Minimum number of WNs in the elastic cluster
default: 0
required: yes
fe_instance_flavor:
type: string
description: front end flavor (num_cpu, memory, disk)
default: medium
wn_instance_flavor:
type: string
description: worker nodes flavor (num_cpu, memory, disk)
default: medium
storage_size:
type: string
description: storage memory required for the instance
default: 50 GB
hybrid:
type: boolean
description: Flag to specify that this cluster will work in an hybrid environment
default: false
required: false
admin_email:
type: string
description: email of the admin user
default: admin@admin.com
admin_api_key:
type: string
description: key to access the API with admin role
default: not_very_secret_api_key
version:
type: string
description: galaxy version to install
default: release_17.05
instance_description:
type: string
description: galaxy instance description
default: "ELIXIR-ITALY"
instance_key_pub:
type: string
description: galaxy instance ssh public key
default: your_ssh_public_key
export_dir:
type: string
description: path to store galaxy data
default: /export
storage_encryption:
type: boolean
description: Enable storage encryption using Vault to store secrets and LUKS to encrypt
default: False
required: true
vault_url:
type: string
description: Hashicorp Vault server url
default: changeit
required: false
vault_wrapping_token:
type: string
description: Vault Wrapping token to write secret
default: not_a_wrapping_token
required: false
vault_secret_path:
type: string
description: Vault path to store secret
default: "subject/depuuid"
required: false
vault_secret_key:
type: string
description: Vault secret key name
default: secret
required: false
flavor:
type: string
description: Galaxy flavor for tools installation
default: "galaxy-no-tools"
reference_data:
type: boolean
description: Install Reference data
default: true
refdata_cvmfs_configuration:
type: string
description: Configure cvmfs or load preconfigured repository
default: 'cvmfs_preconfigured'
refdata_cvmfs_repository_name:
type: string
description: CernVM-FS repository name
default: 'elixir-italy.galaxy.refdata'
refdata_cvmfs_server_url:
type: string
description: CernVM-FS server, replica or stratum-zero
default: '90.147.102.186'
refdata_cvmfs_key_file:
type: string
description: CernVM-FS public key
default: 'elixir-italy.galaxy.refdata.pub'
refdata_cvmfs_proxy_url:
type: string
description: CernVM-FS proxy url
default: 'DIRECT'
refdata_cvmfs_proxy_port:
type: integer
description: CernVM-FS proxy port
default: 80
refdata_dir:
type: string
description: path to store galaxy reference data
default: /cvmfs
node_templates:
elastic_cluster_front_end:
type: tosca.nodes.indigo.ElasticCluster
properties:
deployment_id: orchestrator_deployment_id
iam_access_token: iam_access_token
iam_clues_client_id: iam_clues_client_id
iam_clues_client_secret: iam_clues_client_secret
hybrid: { get_input: hybrid }
requirements:
- lrms: lrms_front_end
- wn: wn_node
galaxy_portal:
type: tosca.nodes.indigo.GalaxyPortalAndStorage
properties:
storage_encryption: { get_input: storage_encryption }
vault_url: { get_input: vault_url }
vault_wrapping_token: { get_input: vault_wrapping_token }
vault_secret_path: { get_input: vault_secret_path }
vault_secret_key: { get_input: vault_secret_key }
admin_email: { get_input: admin_email }
admin_api_key: { get_input: admin_api_key }
version: { get_input: version }
instance_description: { get_input: instance_description }
instance_key_pub: { get_input: instance_key_pub }
export_dir: { get_input: export_dir }
requirements:
- lrms: lrms_front_end
galaxy_tools:
type: tosca.nodes.indigo.GalaxyShedTool
properties:
flavor: { get_input: flavor }
admin_api_key: { get_input: admin_api_key }
version: { get_input: version }
requirements:
- host: galaxy_portal
galaxy_refdata:
type: tosca.nodes.indigo.GalaxyReferenceData
properties:
reference_data: { get_input: reference_data }
refdata_cvmfs_configuration: { get_input: refdata_cvmfs_configuration }
refdata_cvmfs_repository_name: { get_input: refdata_cvmfs_repository_name }
refdata_cvmfs_server_url: { get_input: refdata_cvmfs_server_url }
refdata_cvmfs_key_file: { get_input: refdata_cvmfs_key_file }
refdata_cvmfs_proxy_url: { get_input: refdata_cvmfs_proxy_url }
refdata_cvmfs_proxy_port: { get_input: refdata_cvmfs_proxy_port }
refdata_dir: { get_input: refdata_dir }
flavor: { get_input: flavor }
requirements:
- host: galaxy_portal
- dependency: galaxy_tools
lrms_front_end:
type: tosca.nodes.indigo.LRMS.FrontEnd.SlurmGalaxy
properties:
wn_ips: { get_attribute: [ lrms_wn, private_address ] }
hybrid: { get_input: hybrid }
requirements:
- host: lrms_server
lrms_server:
type: tosca.nodes.indigo.Compute
capabilities:
endpoint:
properties:
dns_name: slurmserver
network_name: PUBLIC
ports:
galaxyctl_api_port:
protocol: tcp
source: 5001
luks_api_port:
protocol: tcp
source: 5000
http_port:
protocol: tcp
source: 80
ftp_port:
protocol: tcp
source: 21
ftp_passive:
protocol: tcp
source_range: [ 30000, 30009 ]
host:
properties:
instance_type: { get_input: fe_instance_flavor }
os:
properties:
image: centos-7-galaxy-base
#image: centos-7_2nic
#image: ubuntu-16.04_2nic
requirements:
# contextually this can only be a relationship type
- local_storage:
# capability is provided by Compute Node Type
node: my_block_storage
capability: tosca.capabilities.Attachment
relationship:
type: tosca.relationships.AttachesTo
properties:
location: { get_input: export_dir }
device: hdb
my_block_storage:
type: tosca.nodes.BlockStorage
properties:
size: { get_input: storage_size }
wn_node:
type: tosca.nodes.indigo.LRMS.WorkerNode.SlurmGalaxy
properties:
front_end_ip: { get_attribute: [ lrms_server, private_address, 0 ] }
public_front_end_ip: { get_attribute: [ lrms_server, public_address, 0 ] }
hybrid: { get_input: hybrid }
capabilities:
wn:
properties:
max_instances: { get_input: wn_num_max }
min_instances: { get_input: wn_num_min }
requirements:
- host: lrms_wn
# This component install slurm.
# Galaxy send jobs to slurm as {{ galaxy }} user.
# So the user has to be created before the slurm installation.
# Galaxy user creation is done in galaxy_wn.
- dependency: galaxy_wn
galaxy_wn:
type: tosca.nodes.indigo.GalaxyWNCVMFS
properties:
export_dir: { get_input: export_dir }
front_end_ip: { get_attribute: [ lrms_server, private_address, 0 ] }
storage_encryption: { get_input: storage_encryption }
reference_data: { get_input: reference_data }
refdata_cvmfs_configuration: { get_input: refdata_cvmfs_configuration }
refdata_cvmfs_repository_name: { get_input: refdata_cvmfs_repository_name }
refdata_cvmfs_server_url: { get_input: refdata_cvmfs_server_url }
refdata_cvmfs_key_file: { get_input: refdata_cvmfs_key_file }
refdata_cvmfs_proxy_url: { get_input: refdata_cvmfs_proxy_url }
refdata_cvmfs_proxy_port: { get_input: refdata_cvmfs_proxy_port }
refdata_dir: { get_input: refdata_dir }
requirements:
- host: lrms_wn
# The worker node has to be configured after the galaxy_portal component
# since the {{ export_dir }} and the external volume is created there.
# This is mandatory for wn_num_min > 0, while can be ignored for pure elastic clusters.
- dependency: galaxy_portal
lrms_wn:
type: tosca.nodes.indigo.Compute
capabilities:
scalable:
properties:
count: { get_input: wn_num_min }
endpoint:
properties:
# Needed on ReCaS Cloud.
# The name of the private network is hard-coded here to append the right interface.
# This configuration broke the portability.
network_name: private_net
host:
properties:
instance_type: { get_input: wn_instance_flavor }
os:
properties:
image: centos-7-galaxy-base
#image: centos-7_2nic
#image: ubuntu-16.04_2nic
outputs:
endpoint:
value: { concat: [ 'http://', get_attribute: [ lrms_server, public_address, 0 ], '/galaxy' ] }
cluster_ip:
value: { get_attribute: [ lrms_server, public_address, 0 ] }
wn_ips:
value: { get_attribute: [ lrms_wn, private_address ] }