Skip to content

Commit

Permalink
Merge branch 'master' into galaxy_job_radar
Browse files Browse the repository at this point in the history
  • Loading branch information
sanjaysrikakulam authored Feb 26, 2025
2 parents f4b27d5 + f3a0d4e commit 34c2721
Show file tree
Hide file tree
Showing 42 changed files with 815 additions and 433 deletions.
60 changes: 60 additions & 0 deletions dnbd3.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,60 @@
---
- name: DNBD3 Infrastructure
hosts: dnbd3primary,dnbd3proxy
become: true
pre_tasks:
- name: Install dependencies
ansible.builtin.package:
name:
- git
state: present

- name: Create dnbd3 user
ansible.builtin.group:
name: "{{ dnbd3_group }}"
state: present
- name: Create dnbd3 user
ansible.builtin.user:
name: "{{ dnbd3_user }}"
group: "{{ dnbd3_group }}"
shell: /bin/false
create_home: false
- name: Grant access to dnbd3 root directory
ansible.builtin.file:
mode: "770"
path: "{{ dnbd3_base_path }}"
state: directory
owner: "{{ dnbd3_user }}"
group: "{{ dnbd3_group }}"
- name: Grant access to tftp root directory
ansible.builtin.file:
mode: "770"
path: "{{ tftpboot_path }}"
state: directory
owner: "{{ tftp_user }}"
group: "{{ tftp_group }}"
when: "'dnbd3primary' in group_names"
roles:
# - role: usegalaxy_eu.handy.os_setup
# become: true
# vars:
# hostname: "{{ inventory_hostname }}"
# enable_hostname: true
# enable_powertools: true
# when: "'dnbd3_primary' in group_names"
- role: usegalaxy_eu.dnbd3
- role: usegalaxy_eu.tftp
when: "'dnbd3primary' in group_names"
- role: geerlingguy.apache
when: "'dnbd3primary' in group_names"


post_tasks:
- name: Grant access to http directory
ansible.builtin.file:
mode: "770"
path: "{{ apache_vhosts[0].documentroot }}"
state: directory
owner: "www-data"
group: "www-data"
when: "'dnbd3primary' in group_names"
2 changes: 2 additions & 0 deletions files/galaxy/config/email_domain_blocklist.conf
Original file line number Diff line number Diff line change
Expand Up @@ -912,6 +912,7 @@ e-mail.com
e-mail.igg.biz
e-mail.org
e-marketstore.ru
e-record.com
e-tomarigi.com
e3z.de
e4ward.com
Expand All @@ -927,6 +928,7 @@ ecallheandi.com
ecolo-online.fr
edgex.ru
edinburgh-airporthotels.com
edny.net
edv.to
ee1.pl
ee2.pl
Expand Down
26 changes: 26 additions & 0 deletions files/galaxy/subdomains/microbiology/static/welcome.html
Original file line number Diff line number Diff line change
@@ -0,0 +1,26 @@
<?xml version="1.0" encoding="utf-8" ?>
<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd">
<html xmlns="http://www.w3.org/1999/xhtml" xml:lang="en" lang="en">
<head>
<style>
body, html { width:100% ;
height:100% ;
overflow:hidden ;
}
iframe { width:100% ;
height:100% ;
border:none ;
position: absolute;
top: -9999em;
visibility: hidden;
}
</style>
</head>
<body style="margin:0px">
<iframe id='center-iframe' src="" height="100%" onload="this.style.position='static'; this.style.visibility='visible';"></iframe>
<script type="text/javascript">
var pseudo_rand = parseInt(Date.now() / 1000 / 60 / 10);
document.getElementById('center-iframe').src = 'https://labs.usegalaxy.org.au/?content_root=https://github.com/galaxyproject/galaxy_codex/blob/main/communities/microgalaxy/lab/usegalaxy.eu.yml';
</script>
</body>
</html>
9 changes: 9 additions & 0 deletions files/galaxy/tpv/interactive_tools.yml
Original file line number Diff line number Diff line change
Expand Up @@ -80,6 +80,11 @@ tools:
inherits: interactive_tool
cores: 1
mem: 4
params:
docker_run_extra_arguments: "-e NB_UID=999"
env:
HOME: /home/jovyan

interactive_tool_divand:
inherits: interactive_tool
cores: 1
Expand Down Expand Up @@ -214,6 +219,10 @@ tools:
inherits: interactive_tool
cores: 1
mem: 4
interactive_tool_cellxgene_vip:
inherits: interactive_tool
cores: 1
mem: 4
interactive_tool_bam_iobio:
inherits: interactive_tool
cores: 1
Expand Down
2 changes: 2 additions & 0 deletions files/galaxy/tpv/tool_defaults.yml
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,8 @@ tools:
env:
- name: GALAXY_MEMORY_MB
value: "{int(mem * 1024)}" # set 5/2023 might be moved to runner or tool wrappers, related to Galaxy issue 15952
- name: _JAVA_OPTIONS
value: -Xmx{round(mem*0.9*1024)}m -Xms256m -Duser.home=/data/2/galaxy_db/tmp -Djava.io.tmpdir=$_GALAXY_JOB_TMP_DIR
- name: HDF5_USE_FILE_LOCKING
value: "FALSE"
- name: SINGULARITYENV_HDF5_USE_FILE_LOCKING
Expand Down
69 changes: 18 additions & 51 deletions files/galaxy/tpv/tools.yml
Original file line number Diff line number Diff line change
Expand Up @@ -221,6 +221,14 @@ tools:
- if: input_size >= 0.01
cores: 3

toolshed.g2.bx.psu.edu/repos/iuc/ncbi_fcs_gx/ncbi_fcs_gx/.*:
rules:
- id: ncbi_fcs_gx
if: |
helpers.job_args_match(job, app, {"mode": {"mode_selector": "screen"}})
mem: 480
cores: 64

toolshed.g2.bx.psu.edu/repos/computational-metabolomics/sirius_csifingerid/sirius_csifingerid/.*:
cores: 8
mem: 8
Expand Down Expand Up @@ -952,14 +960,7 @@ tools:
scheduling:
prefer:
- condor-tpv
env:
_JAVA_OPTIONS: -XX:MaxPermSize=2G -Xmx{int(mem)}G -Xms1G -Djava.io.tmpdir=/data/2/galaxy_db/tmp -Duser.home=/data/2/galaxy_db/tmp
rules:
- if: helpers.tool_version_gte(tool, '2.15.1')
# see usegalaxy-eu/issues#473: https://github.com/usegalaxy-eu/issues/issues/473
env:
_JAVA_OPTIONS: -Xmx{int(mem)}G -Xms1G -Djava.io.tmpdir=/data/2/galaxy_db/tmp -Duser.home=/data/2/galaxy_db/tmp

- if: input_size < 0.1
cores: 1
mem: 4
Expand Down Expand Up @@ -1041,8 +1042,6 @@ tools:
inherits: basic_docker_tool
params:
docker_run_extra_arguments: --user 999
env:
_JAVA_OPTIONS: -Xmx{int(mem)}G -Xms1G

toolshed.g2.bx.psu.edu/repos/bgruening/diamond/diamond/.*:
cores: 6
Expand Down Expand Up @@ -1115,12 +1114,6 @@ tools:
inherits: basic_numba_tool
toolshed.g2.bx.psu.edu/repos/computational-metabolomics/dimspy_align_samples/dimspy_align_samples/.*:
inherits: basic_numba_tool
toolshed.g2.bx.psu.edu/repos/galaxyp/openms_msgfplusadapter/MSGFPlusAdapter/.*:
env:
_JAVA_OPTIONS: -XX:MaxPermSize=2G -Xmx6G -Xms1G -Djava.io.tmpdir=/data/2/galaxy_db/tmp -Duser.home=/data/2/galaxy_db/tmp
toolshed.g2.bx.psu.edu/repos/iracooke/msgfplus/msgfplus/.*:
env:
_JAVA_OPTIONS: -XX:MaxPermSize=2G -Xmx6G -Xms1G -Djava.io.tmpdir=/data/2/galaxy_db/tmp -Duser.home=/data/2/galaxy_db/tmp
toolshed.g2.bx.psu.edu/repos/bgruening/repeat_masker/repeatmasker_wrapper/.*:
rules:
- if: helpers.tool_version_gte(tool, '4.1.5')
Expand All @@ -1131,29 +1124,6 @@ tools:
RM_LIB_PATH: "/data/db/databases/dfam/3.4/"
- if: helpers.tool_version_gte(tool, '4.1.5')
cores: 4
toolshed.g2.bx.psu.edu/repos/galaxyp/reactome_pathwaymatcher/reactome_pathwaymatcher/.*:
env:
_JAVA_OPTIONS: -XX:MaxPermSize=2G -Xmx17G -Xms1G -Djava.io.tmpdir=/data/2/galaxy_db/tmp

toolshed.g2.bx.psu.edu/repos/iuc/bbtools_callvariants/bbtools_callvariants/.*:
env:
_JAVA_OPTIONS: -XX:MaxPermSize=2G -Xmx15G -Xms1G -Djava.io.tmpdir=/data/2/galaxy_db/tmp

toolshed.g2.bx.psu.edu/repos/crs4/prokka/prokka/.*:
env:
_JAVA_OPTIONS: -XX:MaxPermSize=2G -Xmx15G -Xms1G -Djava.io.tmpdir=/data/2/galaxy_db/tmp

toolshed.g2.bx.psu.edu/repos/devteam/picard/picard_SortSam/.*:
env:
_JAVA_OPTIONS: -Xmx4G -Xms1G -Djava.io.tmpdir=/data/2/galaxy_db/tmp

toolshed.g2.bx.psu.edu/repos/galaxyp/peptideshaker/peptide_shaker/.*:
env:
_JAVA_OPTIONS: -XX:MaxPermSize=4G -Xmx120G -Xms4G -Djava.io.tmpdir=/data/2/galaxy_db/tmp

toolshed.g2.bx.psu.edu/repos/iuc/migmap/migmap/.*:
env:
_JAVA_OPTIONS: -XX:MaxPermSize=6G -Xmx14G -Xms1G -Djava.io.tmpdir=/data/2/galaxy_db/tmp

toolshed.g2.bx.psu.edu/repos/iuc/scanpy_cluster_reduce_dimension/scanpy_cluster_reduce_dimension/.*:
inherits: basic_numba_tool
Expand All @@ -1173,24 +1143,11 @@ tools:
toolshed.g2.bx.psu.edu/repos/iuc/scanpy_plot/scanpy_plot/.*:
inherits: basic_numba_tool

toolshed.g2.bx.psu.edu/repos/devteam/sam_merge/sam_merge2/.*:
env:
_JAVA_OPTIONS: -XX:MaxPermSize=2G -Xmx15G -Xms1G -Djava.io.tmpdir=/data/2/galaxy_db/tmp

toolshed.g2.bx.psu.edu/repos/galaxyp/peptideshaker/search_gui/.*:
env:
_JAVA_OPTIONS: -XX:MaxPermSize=2G -Xmx6G -Xms1G -Djava.io.tmpdir=/data/2/galaxy_db/tmp -Duser.home=/data/2/galaxy_db/tmp

toolshed.g2.bx.psu.edu/repos/pjbriggs/trimmomatic/trimmomatic/.*:
env:
_JAVA_OPTIONS: -Xmx6G -Xms1G -Djava.io.tmpdir=/data/2/galaxy_db/tmp -Duser.home=/data/2/galaxy_db/tmp

toolshed.g2.bx.psu.edu/repos/iuc/unicycler/unicycler/.*:
cores: 24
mem: 80
env:
TERM: vt100
_JAVA_OPTIONS: -XX:MaxPermSize=2G -Xmx6G -Xmx{int(mem)}G -Xms1G -Djava.io.tmpdir=/data/2/galaxy_db/tmp -Duser.home=/data/2/galaxy_db/tmp
scheduling:
accept:
- pulsar
Expand Down Expand Up @@ -1566,3 +1523,13 @@ tools:
toolshed.g2.bx.psu.edu/repos/iuc/optitype/optitype/.*:
cores: 12
mem: 256

# ppanggolin all tries to remove TMP folders, on NFS this does not always work, so we run this tool in singularity with internal (non-NFS) TMP
toolshed.g2.bx.psu.edu/repos/iuc/ppanggolin_all/ppanggolin_all/.*:
env:
_GALAXY_JOB_TMP_DIR: '/tmp'
params:
singularity_no_mount: null
scheduling:
require:
- singularity
9 changes: 9 additions & 0 deletions files/traefik/rules/mq-router.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,9 @@
tcp:

Check warning on line 1 in files/traefik/rules/mq-router.yml

View workflow job for this annotation

GitHub Actions / Lint

1:1 [document-start] missing document start "---"
routers:
mq-rtr:
rule: "HostSNI(`*`)" # || ClientIP(`132.230.224.103`)" Allow by host IP
service: "mq"
entryPoints:
- amqps
tls:
passthrough: true
6 changes: 6 additions & 0 deletions files/traefik/rules/mq-service.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,6 @@
tcp:

Check warning on line 1 in files/traefik/rules/mq-service.yml

View workflow job for this annotation

GitHub Actions / Lint

1:1 [document-start] missing document start "---"
services:
mq:
loadBalancer:
servers:
- address: "10.5.68.232:5671" #replace once mq02 is in playbook
1 change: 1 addition & 0 deletions files/traefik/rules/template-subdomains.yml
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,7 @@ http:
{{template "subdomain" "metagenomics"}}
{{template "subdomain" "microbiome"}}
{{template "subdomain" "microgalaxy"}}
{{template "subdomain" "microbiology"}}
{{template "subdomain" "rna"}}
{{template "subdomain" "graphclust"}}
{{template "subdomain" "ecology"}}
Expand Down
2 changes: 1 addition & 1 deletion group_vars/all.yml
Original file line number Diff line number Diff line change
Expand Up @@ -38,7 +38,7 @@ nginx_selinux_allow_local_connections: true
# Allow root login on all machines - TBD
ssh_allow_root_with_key: true

redis_connection_string: "redis://:{{ redis_requirepass }}@mq.galaxyproject.eu:6379/0"
redis_connection_string: "redis://:{{ redis_requirepass }}@mq02.galaxyproject.eu:6379/0"
# Chrony
chrony_server: 'time.ruf.uni-freiburg.de iburst'
chrony_acquisitionport: 0
Expand Down
15 changes: 15 additions & 0 deletions group_vars/dnbd3/vars.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,15 @@
---
# --- dnbd3 setup vars ---
#dnbd3_base_path: "/mnt/s3/dnbd3"
dnbd3_git_tag: "master"
dnbd3_git_repo: "git://git.openslx.org/dnbd3.git"
# --- dnbd3 config vars ---
dnbd3_listen_port: "5003"
dnbd3_client_penalty: "100000"
dnbd3_primary: "{{ hostvars['dnbd3-primary.galaxyproject.eu']['ansible_default_ipv4']['address'] }}"
dnbd3_primary_comment: "Primary DNBD3 server, which replicates S3FS mounted QCOW2 images."
# dnbd3_primary_backup: "10.20.56.174"
# dnbd3_primary_backup_comment: "Secondary/backup DNBD3 server, which replicates S3FS mounted QCOW2 images."
dnbd3_poxys:
- ip: "{{ hostvars['dnbd3-proxy.galaxyproject.eu']['ansible_default_ipv4']['address'] }}"
comment: "1st DNBD3 Proxy"
10 changes: 10 additions & 0 deletions group_vars/dnbd3primary/vars.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,10 @@
---
dnbd3_is_proxy: false
dnbd3_base_path: "/export/dnbd3-cache/images"
tftpboot_path: "/export/dnbd3-cache/boot"
apache_listen_port: 80
apache_remove_default_vhost: true
apache_vhosts:
- servername: "dnbd3-primary.galaxyproject.eu"
serveralias: "{{ hostvars['dnbd3-primary.galaxyproject.eu']['ansible_default_ipv4']['address'] }}"
documentroot: "/export/dnbd3-cache/http"
3 changes: 3 additions & 0 deletions group_vars/dnbd3proxy/vars.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,3 @@
---
dnbd3_is_proxy: true
dnbd3_base_path: "/mnt/dnbd3"
9 changes: 5 additions & 4 deletions group_vars/gxconfig.yml
Original file line number Diff line number Diff line change
Expand Up @@ -1011,7 +1011,7 @@ base_app_main: &BASE_APP_MAIN
# option ``interactivetoolsproxy_map``, which overrides this one.
# The value of this option will be resolved with respect to
# <data_dir>.
interactivetools_map: "{{ galaxy_mutable_data_dir }}/interactivetools_map.sqlite"
#interactivetools_map: interactivetools_map.sqlite

# Use a database supported by SQLAlchemy as map for the
# interactivetool proxy. When this option is set, the value of
Expand All @@ -1021,7 +1021,7 @@ base_app_main: &BASE_APP_MAIN
# Mappings are written to the table "gxitproxy" within the database.
# This value cannot match ``database_connection`` nor
# ``install_database_connection``.
#interactivetoolsproxy_map: null
interactivetoolsproxy_map: "{{ interactivetools_db_connection }}"

# Prefix to use in the formation of the subdomain or path for
# interactive tools
Expand Down Expand Up @@ -1062,6 +1062,7 @@ base_app_main: &BASE_APP_MAIN
'metagenomics.usegalaxy.eu': Metagenomics
'microbiome.usegalaxy.eu': Microbiome
'microgalaxy.usegalaxy.eu': microGalaxy
'microbiology.usegalaxy.eu': Microbiology
'rna.usegalaxy.eu': RNA
'graphclust.usegalaxy.eu': GraphClust
'ecology.usegalaxy.eu': Ecology
Expand Down Expand Up @@ -2589,7 +2590,7 @@ base_app_main: &BASE_APP_MAIN

# API key for OpenAI (https://openai.com/) to enable the wizard (or
# more?)
#openai_api_key: null
openai_api_key: "{{ openai_api_key }}"

# OpenAI model to enable the wizard.
#openai_model: gpt-4o
Expand Down Expand Up @@ -2707,7 +2708,7 @@ base_app_main: &BASE_APP_MAIN

# Directory to store temporary files for file sources. This defaults
# to new_file_path if not set.
#file_source_temp_dir: null
file_source_temp_dir: /data/jwd04/tmp/

# Default value for use_temp_files for webdav plugins that don't
# explicitly declare this.
Expand Down
Loading

0 comments on commit 34c2721

Please sign in to comment.