From ea9e1c125e414f00c19f750bc5ed94e52d4fd170 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jos=C3=A9=20Manuel=20Dom=C3=ADnguez?= Date: Thu, 4 Jul 2024 14:38:31 +0200 Subject: [PATCH 01/62] Store interactive tool sessions in PostgreSQL table "gxitproxy" --- group_vars/gxconfig.yml | 11 +++++++---- group_vars/sn06.yml | 6 +++--- group_vars/sn07.yml | 6 +++--- 3 files changed, 13 insertions(+), 10 deletions(-) diff --git a/group_vars/gxconfig.yml b/group_vars/gxconfig.yml index db7fb2b6b..b5d65d7b8 100644 --- a/group_vars/gxconfig.yml +++ b/group_vars/gxconfig.yml @@ -958,10 +958,13 @@ base_app_main: &BASE_APP_MAIN # subdomain. Defaults to "/". #interactivetools_base_path: / - # Map for interactivetool proxy. - # The value of this option will be resolved with respect to - # . - interactivetools_map: "{{ galaxy_mutable_data_dir }}/interactivetools_map.sqlite" + # Map for interactivetool proxy. It may be either a path to a SQLite + # database or a SQLAlchemy database URL (see + # https://docs.sqlalchemy.org/en/20/core/engines.html#database-urls). + # In either case, mappings will be written to the table "gxitproxy" within the + # database. If it is a path, the value of this option will be resolved with + # respect to . + interactivetools_map: "{{ galaxy_db_connection }}" # Prefix to use in the formation of the subdomain or path for # interactive tools diff --git a/group_vars/sn06.yml b/group_vars/sn06.yml index 58de5f569..271263a26 100644 --- a/group_vars/sn06.yml +++ b/group_vars/sn06.yml @@ -226,13 +226,13 @@ galaxy_systemd_memory_limit_workflow: 15 # gie_proxy gie_proxy_dir: "{{ galaxy_root }}/gie-proxy/proxy" -gie_proxy_git_version: main +gie_proxy_git_version: v0.1.0 gie_proxy_setup_nodejs: nodeenv gie_proxy_virtualenv_command: "{{ conda_prefix }}/envs/_galaxy_/bin/python -m venv --copies" #"{{ pip_virtualenv_command }}" -gie_proxy_nodejs_version: "10.13.0" +gie_proxy_nodejs_version: "14.21.3" gie_proxy_virtualenv: "{{ galaxy_root }}/gie-proxy/venv" gie_proxy_setup_service: systemd -gie_proxy_sessions_path: "{{ galaxy_mutable_data_dir }}/interactivetools_map.sqlite" +gie_proxy_sessions_path: "{{ galaxy_db_connection }}" gie_proxy_port: 8800 gie_proxy_verbose: true diff --git a/group_vars/sn07.yml b/group_vars/sn07.yml index 3eac16461..722a5c654 100644 --- a/group_vars/sn07.yml +++ b/group_vars/sn07.yml @@ -280,13 +280,13 @@ condor_extra: | # gie_proxy gie_proxy_dir: "{{ galaxy_root }}/gie-proxy/proxy" -gie_proxy_git_version: main +gie_proxy_git_version: v0.1.0 gie_proxy_setup_nodejs: nodeenv gie_proxy_virtualenv_command: "{{ conda_prefix }}/envs/_galaxy_/bin/python -m venv --copies" #"{{ pip_virtualenv_command }}" -gie_proxy_nodejs_version: "10.13.0" +gie_proxy_nodejs_version: "14.21.3" gie_proxy_virtualenv: "{{ galaxy_root }}/gie-proxy/venv" gie_proxy_setup_service: systemd -gie_proxy_sessions_path: "{{ galaxy_mutable_data_dir }}/interactivetools_map.sqlite" +gie_proxy_sessions_path: "{{ galaxy_db_connection }}" gie_proxy_port: 8800 gie_proxy_verbose: true From 45b11ec8b58e30cd90d859b4c6f6486ac818c0c1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jos=C3=A9=20Manuel=20Dom=C3=ADnguez?= Date: Fri, 19 Jul 2024 16:49:52 +0200 Subject: [PATCH 02/62] Store interactive tool sessions in PostgreSQL database "gxitproxy" --- group_vars/gxconfig.yml | 24 ++++++++---- group_vars/sn06.yml | 2 +- group_vars/sn07.yml | 2 +- secret_group_vars/db-main.yml | 74 +++++++++++++++++++---------------- 4 files changed, 60 insertions(+), 42 deletions(-) diff --git a/group_vars/gxconfig.yml b/group_vars/gxconfig.yml index b5d65d7b8..4be1ab435 100644 --- a/group_vars/gxconfig.yml +++ b/group_vars/gxconfig.yml @@ -958,13 +958,23 @@ base_app_main: &BASE_APP_MAIN # subdomain. Defaults to "/". #interactivetools_base_path: / - # Map for interactivetool proxy. It may be either a path to a SQLite - # database or a SQLAlchemy database URL (see - # https://docs.sqlalchemy.org/en/20/core/engines.html#database-urls). - # In either case, mappings will be written to the table "gxitproxy" within the - # database. If it is a path, the value of this option will be resolved with - # respect to . - interactivetools_map: "{{ galaxy_db_connection }}" + # Map for the interactivetool proxy. Mappings are stored on a SQLite + # database file located on this path. As an alternative, you may also + # store them on any other RDBMS supported by SQLAlchemy using the + # option ``interactivetools_map_sqlalchemy``. + # The value of this option will be resolved with respect to + # . + #interactivetools_map: interactivetools_map.sqlite + + # Use a database supported by SQLAlchemy as map for the + # interactivetool proxy. When this option is set, the value of + # ``interactivetools_map`` is ignored. The value of this option must + # be a `SQLAlchemy database URL + # `_. + # Mappings are written to the table "gxitproxy" within the database. + # This value cannot match ``database_connection`` nor + # ``install_database_connection``. + interactivetools_map_sqlalchemy: "{{ interactivetools_db_connection }}" # Prefix to use in the formation of the subdomain or path for # interactive tools diff --git a/group_vars/sn06.yml b/group_vars/sn06.yml index 271263a26..58ed7dc4f 100644 --- a/group_vars/sn06.yml +++ b/group_vars/sn06.yml @@ -232,7 +232,7 @@ gie_proxy_virtualenv_command: "{{ conda_prefix }}/envs/_galaxy_/bin/python -m ve gie_proxy_nodejs_version: "14.21.3" gie_proxy_virtualenv: "{{ galaxy_root }}/gie-proxy/venv" gie_proxy_setup_service: systemd -gie_proxy_sessions_path: "{{ galaxy_db_connection }}" +gie_proxy_sessions_path: "{{ interactivetools_db_connection }}" gie_proxy_port: 8800 gie_proxy_verbose: true diff --git a/group_vars/sn07.yml b/group_vars/sn07.yml index 722a5c654..fa4bd25a4 100644 --- a/group_vars/sn07.yml +++ b/group_vars/sn07.yml @@ -286,7 +286,7 @@ gie_proxy_virtualenv_command: "{{ conda_prefix }}/envs/_galaxy_/bin/python -m ve gie_proxy_nodejs_version: "14.21.3" gie_proxy_virtualenv: "{{ galaxy_root }}/gie-proxy/venv" gie_proxy_setup_service: systemd -gie_proxy_sessions_path: "{{ galaxy_db_connection }}" +gie_proxy_sessions_path: "{{ interactivetools_db_connection }}" gie_proxy_port: 8800 gie_proxy_verbose: true diff --git a/secret_group_vars/db-main.yml b/secret_group_vars/db-main.yml index 0629bd046..1dcc2d02b 100644 --- a/secret_group_vars/db-main.yml +++ b/secret_group_vars/db-main.yml @@ -1,34 +1,42 @@ $ANSIBLE_VAULT;1.1;AES256 -32363835343162383736393837653534326237313362343234363037623661386634323939343839 -6434343662383832613038643834396634376135633164660a623032383231383664666531393465 -39616363313136373636616536616634313432363732306538366462383536373764316266626166 -3339666466313036370a333337646531393166646666343963323734323630396366333162626638 -39633139363435626337363235353364623731643965313830303239316265616165333838656562 -39336461626130366139636664323762616330373032326562653630396233656337303966386139 -36366161336563646666343438613834363734613861373737303165323838376564666361633937 -65613833666638303433626537663563653538623962343537363033636166323038626236346262 -31663830343532343430366231393139356264636134303131613132643435343834356336626437 -37373934613035333333383836353762313866343733396133323133663635626561323437653038 -32613236316362613436323961633666363639343135383862353764643639663837636538346162 -37336136386263333534356162383638613761396462393265336339346264306465316565323561 -66343239336562373137303538396230383836343831656535313063663137666365616434366337 -39383337343266326263643637353033623032396461373439323661313134303266663263316635 -32396362393935333963346237633239653936363263313239336130623039356565343034343435 -32656533363864393463373839343662666562616233336262326463316333356262376365636464 -64393931346532336438333564313838666432383434393531306563653335653166633565616137 -32386438623061656434353430326630313736663237326265386133383461333930393266383234 -64393433613832316533303661616334386136373336316137323636396530353365363263663966 -33386233343630336562303062313362313364353762366633323136393264656239633561633761 -39386338396132333862303530316339333531363066393637353263633663636365616261353639 -63643038323062373964376562313139653333396661636265623435613963653966666661333763 -38383663363532316636383961653338653864653361346131633364323863626566616265383238 -33383865623531663135616561643530613932376532336534343139366433383334306434363364 -38353964393466633635316262626339343333633636643265313562353432666262643130336638 -30636264313131653264373237333062323637623439366632353933666330396165663466326533 -62383762663332666432373262633737663365313063643838316163656533376439656438346430 -66656462616437313430663364303434373738346636366439646563663237376532656439373166 -31366436363866383638646435336461396665636430616365643563646566386565383435306566 -38333331386439356264333534353934633035323232623233313935356238343765613362386638 -64383261616466396532393834356263383662376539306662666437333938333434383531393033 -39316137343738616166303861616136636638383338383035623138323437666630303965326237 -6561 +30343163336432356363396364356335373732366664356366653934623361373538653032383566 +3065616462663065663464623332383663643534363738300a323939373733666539386132373366 +37366262316235633030306536373363323736613130393135313137306238646137316632376132 +3138306337396337640a636634633865333231616464366366306331646332643665613832333538 +32363365313836313031363738656661373362613631363763353634636261386536623662323239 +38343939646365353531366536396566313362316562666534306463326436333134636161316637 +64363061346133653839316438626537663036346536613065363734646638373030643931643339 +37303465346465353739306230363639376236393831363630636265306661316135613730653433 +61613661346637363064336363303438313131633766663134633337666461626463343138623539 +65656333343233353538313334326332353131656666393339663735363166393836326135653035 +33323539323064633964656636323761313237386661353162323337396462353962663430326338 +36356636666431336438336333663535636236386139643766313665663731623762623039386439 +32613964616266643565386238326663656433616132333163333731666537366565663065393164 +33346434633132613962363338313135383737303538363731326135636263383865636664633163 +34336364313234326231636563633430646363623630356239653634363565386532613738353330 +38636564393461303136643036346366313263303665333866623433316434633739373166633363 +31336532616334663737323536306565363834353263623061376263653634626239623934316366 +65646231353835643162636362363161313430323536653431333134316338313335623262346330 +39363361376333633664326164316335666532646638646164636530666230393037666266663962 +63323939326636643731333638393862323435386237636464613261633137666137386336626133 +65396634616366326362626162643134383030666361323137393832346133346134303661616265 +66373835313566363762343662623931336663353432303766323033373963626138363038346662 +35353061303665373765653039346234373034306531633332393737353233323233383335646134 +32393235373132356239653736353663636262376432616634346565323438653435326637656664 +39356131343264613931386661306630636364626263623838346661613934303737306437363634 +64613533373161303033613864616435306338336437666661623364656637323863643539376234 +63623165616238633834386133383934373764306263666365333933386633653961323136376164 +62313239396163396661613664666437643361303765303039356463646131663362333533363934 +35343430616235626663656230306263643533623433386637633531663937363737323338303139 +62306663613665353435323466623062653637663634353631323736326663313364623464313335 +32643338343065353666383366633838396563303931303638383864303661303863663564646264 +32653361353332653862633438343336363861386565393530643737396233626163316435366337 +61366138336338653239613935326333303232333639626264346538323266346264643263396439 +39383865656638346463386432376366623635613534386662376533323838643530653439616531 +62656638633634326532623138393030316336383364663438636662333033633562386561653935 +33313430373666613437653334356535633964323065653336633739333261303531653533323065 +62666231633132623335333939376536376265323334663262363364303364613364623431353466 +34343166396266613961313637393332643431366438653930303236653061643466346130343331 +62643232323830316662303030613139326366636332333866393338616465363638363635336634 +30353935633933303938316566656561643532383038356564626265366162666666323966386365 +6366 From c9abf01ddb15f2070b707bb542225afae622940a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jos=C3=A9=20Manuel=20Dom=C3=ADnguez?= <43052541+kysrpex@users.noreply.github.com> Date: Thu, 5 Sep 2024 09:58:06 +0200 Subject: [PATCH 03/62] Rename `interactivetools_map_sqlalchemy` to `interactivetoolsproxy_map` --- group_vars/gxconfig.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/group_vars/gxconfig.yml b/group_vars/gxconfig.yml index 4be1ab435..ed9a4b68e 100644 --- a/group_vars/gxconfig.yml +++ b/group_vars/gxconfig.yml @@ -958,10 +958,10 @@ base_app_main: &BASE_APP_MAIN # subdomain. Defaults to "/". #interactivetools_base_path: / - # Map for the interactivetool proxy. Mappings are stored on a SQLite + # Map for the interactivetool proxy. Mappings are stored in a SQLite # database file located on this path. As an alternative, you may also - # store them on any other RDBMS supported by SQLAlchemy using the - # option ``interactivetools_map_sqlalchemy``. + # store them in any other RDBMS supported by SQLAlchemy using the + # option ``interactivetoolsproxy_map``, which overrides this one. # The value of this option will be resolved with respect to # . #interactivetools_map: interactivetools_map.sqlite @@ -974,7 +974,7 @@ base_app_main: &BASE_APP_MAIN # Mappings are written to the table "gxitproxy" within the database. # This value cannot match ``database_connection`` nor # ``install_database_connection``. - interactivetools_map_sqlalchemy: "{{ interactivetools_db_connection }}" + interactivetoolsproxy_map: "{{ interactivetools_db_connection }}" # Prefix to use in the formation of the subdomain or path for # interactive tools From 7e8a6378207d9d2beecf68d52cfea32f22ed0594 Mon Sep 17 00:00:00 2001 From: Mira Kuntz Date: Wed, 13 Nov 2024 17:23:13 +0100 Subject: [PATCH 04/62] move java env options to default tool and reduce xms --- files/galaxy/tpv/tool_defaults.yml | 5 ++- files/galaxy/tpv/tools.yml | 51 ------------------------------ 2 files changed, 4 insertions(+), 52 deletions(-) diff --git a/files/galaxy/tpv/tool_defaults.yml b/files/galaxy/tpv/tool_defaults.yml index 19d3d0b3c..f880736b4 100644 --- a/files/galaxy/tpv/tool_defaults.yml +++ b/files/galaxy/tpv/tool_defaults.yml @@ -8,7 +8,10 @@ tools: mem: cores * 3.8 gpus: 0 env: - GALAXY_MEMORY_MB: "{int(mem * 1024)}" # set 5/2023 might be moved to runner or tool wrappers, related to Galaxy issue 15952 + - name: GALAXY_MEMORY_MB + value: "{int(mem * 1024)}" # set 5/2023 might be moved to runner or tool wrappers, related to Galaxy issue 15952 + - name: _JAVA_OPTIONS + value: -Xmx{round(mem*0.9*1024)}m -Xms256m -Duser.home=/data/2/galaxy_db/tmp -Djava.io.tmpdir=$_GALAXY_JOB_TMP_DIR params: metadata_strategy: "extended" tmp_dir: true diff --git a/files/galaxy/tpv/tools.yml b/files/galaxy/tpv/tools.yml index b367b1ff6..a28f95ca2 100644 --- a/files/galaxy/tpv/tools.yml +++ b/files/galaxy/tpv/tools.yml @@ -930,14 +930,7 @@ tools: scheduling: prefer: - condor-tpv - env: - _JAVA_OPTIONS: -XX:MaxPermSize=2G -Xmx{int(mem)}G -Xms1G -Djava.io.tmpdir=/data/2/galaxy_db/tmp -Duser.home=/data/2/galaxy_db/tmp rules: - - if: helpers.tool_version_gte(tool, '2.15.1') - # see usegalaxy-eu/issues#473: https://github.com/usegalaxy-eu/issues/issues/473 - env: - _JAVA_OPTIONS: -Xmx{int(mem)}G -Xms1G -Djava.io.tmpdir=/data/2/galaxy_db/tmp -Duser.home=/data/2/galaxy_db/tmp - - if: input_size < 0.1 cores: 1 mem: 4 @@ -1019,8 +1012,6 @@ tools: inherits: basic_docker_tool params: docker_run_extra_arguments: --user 999 - env: - _JAVA_OPTIONS: -Xmx{int(mem)}G -Xms1G toolshed.g2.bx.psu.edu/repos/bgruening/diamond/diamond/.*: cores: 6 @@ -1093,12 +1084,6 @@ tools: inherits: basic_numba_tool toolshed.g2.bx.psu.edu/repos/computational-metabolomics/dimspy_align_samples/dimspy_align_samples/.*: inherits: basic_numba_tool - toolshed.g2.bx.psu.edu/repos/galaxyp/openms_msgfplusadapter/MSGFPlusAdapter/.*: - env: - _JAVA_OPTIONS: -XX:MaxPermSize=2G -Xmx6G -Xms1G -Djava.io.tmpdir=/data/2/galaxy_db/tmp -Duser.home=/data/2/galaxy_db/tmp - toolshed.g2.bx.psu.edu/repos/iracooke/msgfplus/msgfplus/.*: - env: - _JAVA_OPTIONS: -XX:MaxPermSize=2G -Xmx6G -Xms1G -Djava.io.tmpdir=/data/2/galaxy_db/tmp -Duser.home=/data/2/galaxy_db/tmp toolshed.g2.bx.psu.edu/repos/bgruening/repeat_masker/repeatmasker_wrapper/.*: rules: - if: helpers.tool_version_gte(tool, '4.1.5') @@ -1109,29 +1094,6 @@ tools: RM_LIB_PATH: "/data/db/databases/dfam/3.4/" - if: helpers.tool_version_gte(tool, '4.1.5') cores: 4 - toolshed.g2.bx.psu.edu/repos/galaxyp/reactome_pathwaymatcher/reactome_pathwaymatcher/.*: - env: - _JAVA_OPTIONS: -XX:MaxPermSize=2G -Xmx17G -Xms1G -Djava.io.tmpdir=/data/2/galaxy_db/tmp - - toolshed.g2.bx.psu.edu/repos/iuc/bbtools_callvariants/bbtools_callvariants/.*: - env: - _JAVA_OPTIONS: -XX:MaxPermSize=2G -Xmx15G -Xms1G -Djava.io.tmpdir=/data/2/galaxy_db/tmp - - toolshed.g2.bx.psu.edu/repos/crs4/prokka/prokka/.*: - env: - _JAVA_OPTIONS: -XX:MaxPermSize=2G -Xmx15G -Xms1G -Djava.io.tmpdir=/data/2/galaxy_db/tmp - - toolshed.g2.bx.psu.edu/repos/devteam/picard/picard_SortSam/.*: - env: - _JAVA_OPTIONS: -Xmx4G -Xms1G -Djava.io.tmpdir=/data/2/galaxy_db/tmp - - toolshed.g2.bx.psu.edu/repos/galaxyp/peptideshaker/peptide_shaker/.*: - env: - _JAVA_OPTIONS: -XX:MaxPermSize=4G -Xmx120G -Xms4G -Djava.io.tmpdir=/data/2/galaxy_db/tmp - - toolshed.g2.bx.psu.edu/repos/iuc/migmap/migmap/.*: - env: - _JAVA_OPTIONS: -XX:MaxPermSize=6G -Xmx14G -Xms1G -Djava.io.tmpdir=/data/2/galaxy_db/tmp toolshed.g2.bx.psu.edu/repos/iuc/scanpy_cluster_reduce_dimension/scanpy_cluster_reduce_dimension/.*: inherits: basic_numba_tool @@ -1151,24 +1113,11 @@ tools: toolshed.g2.bx.psu.edu/repos/iuc/scanpy_plot/scanpy_plot/.*: inherits: basic_numba_tool - toolshed.g2.bx.psu.edu/repos/devteam/sam_merge/sam_merge2/.*: - env: - _JAVA_OPTIONS: -XX:MaxPermSize=2G -Xmx15G -Xms1G -Djava.io.tmpdir=/data/2/galaxy_db/tmp - - toolshed.g2.bx.psu.edu/repos/galaxyp/peptideshaker/search_gui/.*: - env: - _JAVA_OPTIONS: -XX:MaxPermSize=2G -Xmx6G -Xms1G -Djava.io.tmpdir=/data/2/galaxy_db/tmp -Duser.home=/data/2/galaxy_db/tmp - - toolshed.g2.bx.psu.edu/repos/pjbriggs/trimmomatic/trimmomatic/.*: - env: - _JAVA_OPTIONS: -Xmx6G -Xms1G -Djava.io.tmpdir=/data/2/galaxy_db/tmp -Duser.home=/data/2/galaxy_db/tmp - toolshed.g2.bx.psu.edu/repos/iuc/unicycler/unicycler/.*: cores: 24 mem: 80 env: TERM: vt100 - _JAVA_OPTIONS: -XX:MaxPermSize=2G -Xmx6G -Xmx{int(mem)}G -Xms1G -Djava.io.tmpdir=/data/2/galaxy_db/tmp -Duser.home=/data/2/galaxy_db/tmp scheduling: accept: - pulsar From 940077492259067be2bd93c3dfbd966dcd9fab12 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B6rn=20Gr=C3=BCning?= Date: Wed, 11 Dec 2024 18:27:37 +0100 Subject: [PATCH 05/62] add ncbi_fcs_gx --- files/galaxy/tpv/tools.yml | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/files/galaxy/tpv/tools.yml b/files/galaxy/tpv/tools.yml index 12a171e67..b9879a378 100644 --- a/files/galaxy/tpv/tools.yml +++ b/files/galaxy/tpv/tools.yml @@ -212,6 +212,14 @@ tools: - if: input_size >= 0.01 cores: 3 + toolshed.g2.bx.psu.edu/repos/iuc/ncbi_fcs_gx/ncbi_fcs_gx/.*: + rules: + - id: ncbi_fcs_gx + if: | + helpers.job_args_match(job, app, {"mode": {"mode_selector": "screen"}}) + mem: 480 + cores: 24 + toolshed.g2.bx.psu.edu/repos/computational-metabolomics/sirius_csifingerid/sirius_csifingerid/.*: cores: 8 mem: 8 From abf009338bea69fb78e1b14a7274cfb216278ffe Mon Sep 17 00:00:00 2001 From: Mira Kuntz Date: Wed, 8 Jan 2025 15:23:06 +0100 Subject: [PATCH 06/62] add dnbd3 playbook --- dnbd3.yml | 55 +++++++++++++++++++++++++++++++ group_vars/dnbd3/vars.yml | 37 +++++++++++++++++++++ group_vars/dnbd3_primary/vars.yml | 8 +++++ group_vars/dnbd3_proxy/vars.yml | 3 ++ hosts | 10 ++++++ requirements.yaml | 8 +++++ 6 files changed, 121 insertions(+) create mode 100644 dnbd3.yml create mode 100644 group_vars/dnbd3/vars.yml create mode 100644 group_vars/dnbd3_primary/vars.yml create mode 100644 group_vars/dnbd3_proxy/vars.yml diff --git a/dnbd3.yml b/dnbd3.yml new file mode 100644 index 000000000..3bf98ddc4 --- /dev/null +++ b/dnbd3.yml @@ -0,0 +1,55 @@ +--- +- hosts: + - dnbd3-primary + - dnbd3-proxy + become: true + pre_tasks: + - name: Create dnbd3 user + ansible.builtin.group: + name: "{{ dnbd3_group }}" + state: present + - name: Create dnbd3 user + ansible.builtin.user: + name: "{{ dnbd3_user }}" + group: "{{ dnbd3_group }}" + shell: /bin/false + create_home: false + - name: Grant access to dnbd3 root directory + ansible.builtin.file: + mode: "770" + path: "{{ dnbd3_base_path }}" + state: directory + owner: "{{ dnbd3_user }}" + group: "{{ dnbd3_group }}" + - name: Grant access to http directory + ansible.builtin.file: + mode: "770" + state: directory + path: "{{ tftpboot_path }}" + owner: "{{ dnbd3_user }}" + group: "{{ dnbd3_group }}" + when: "'dndb3_primary' in group_names" + roles: + # - role: usegalaxy_eu.handy.os_setup + # become: true + # vars: + # hostname: "{{ inventory_hostname }}" + # enable_hostname: true + # enable_powertools: true + # when: "'dndb3_primary' in group_names" + - role: usegalaxy_eu.dnbd3 + - role: usegalaxy_eu.tftp + when: "'dndb3_primary' in group_names" + - role: geerlingguy.apache + when: "'dndb3_primary' in group_names" + + + post_tasks: + - name: Grant access to http directory + ansible.builtin.file: + mode: "770" + path: "{{ apache_vhosts[0].documentroot }}" + state: directory + owner: "www-data" + group: "www-data" + when: "'dndb3_primary' in group_names" diff --git a/group_vars/dnbd3/vars.yml b/group_vars/dnbd3/vars.yml new file mode 100644 index 000000000..708c9fba1 --- /dev/null +++ b/group_vars/dnbd3/vars.yml @@ -0,0 +1,37 @@ +--- +# --- dnbd3 setup vars --- +#dnbd3_base_path: "/mnt/s3/dnbd3" +dnbd3_log_file: "/tmp/dnbd3-server.log" +dnbd3_config_path: "/usr/local/etc/dnbd3-server" +dnbd3_user: "dnbd3" +dnbd3_group: "dnbd3" +dnbd3_server_conf: "{{ dnbd3_base_path }}/server.conf" +dnbd3_alt_servers_conf: "{{ dnbd3_base_path }}/alt-servers" +# --- dnbd3 repo vars --- +dnbd3_git_tag: "master" +dnbd3_git_repo: "git://git.openslx.org/dnbd3.git" +# --- dnbd3 config vars --- +dnbd3_listen_port: "5003" +dnbd3_client_penalty: "100000" +dnbd3_primary: "{{ hostvars[dnbd3_primary]['ansible_default_ipv4']['address'] }}" +dnbd3_primary_comment: "Primary DNBD3 server, which replicates S3FS mounted QCOW2 images." +# dnbd3_primary_backup: "10.20.56.174" +# dnbd3_primary_backup_comment: "Secondary/backup DNBD3 server, which replicates S3FS mounted QCOW2 images." +dnbd3_poxys: + - ip: "{{ hostvars[dnbd3_proxy]['ansible_default_ipv4']['address'] }}" + comment: "1st DNBD3 Proxy" + + +# --- dnbd3 build vars --- +dnbd3_src_dir: "/usr/local/src/dnbd3" +dnbd3_build_dir: "{{ dnbd3_src_dir }}/build" +dnbd3_server_binary: "/usr/local/sbin/dnbd3-server" +dnbd3_new_binary: "{{ dnbd3_src_dir }}/dnbd3-server.new" +dnbd3_old_binary: "{{ dnbd3_src_dir }}/dnbd3-server.old" + +# --- dnbd3 systemd vars --- +dnbd3_systemd_dir: "/etc/systemd/system" +dnbd3_service_dir: "{{ dnbd3_systemd_dir }}/dnbd3-server.service.d" +dnbd3_service_file: "{{ dnbd3_systemd_dir }}/dnbd3-server.service" +dnbd3_limits_file: "{{ dnbd3_service_dir }}/limits.conf" +dnbd3_srv_mount_file: "{{ dnbd3_systemd_dir }}/srv.mount" diff --git a/group_vars/dnbd3_primary/vars.yml b/group_vars/dnbd3_primary/vars.yml new file mode 100644 index 000000000..81ec927bf --- /dev/null +++ b/group_vars/dnbd3_primary/vars.yml @@ -0,0 +1,8 @@ +--- +dnbd3_is_proxy: false +dnbd3_base_path: "/export/dnbd3-cache/images" +tftpboot_path: "/export/dnbd3-cache/boot" +apache_listen_port: 80 +apache_vhosts: + - servername: "example.com" + documentroot: "/export/dnbd3-cache/http" diff --git a/group_vars/dnbd3_proxy/vars.yml b/group_vars/dnbd3_proxy/vars.yml new file mode 100644 index 000000000..c758024c4 --- /dev/null +++ b/group_vars/dnbd3_proxy/vars.yml @@ -0,0 +1,3 @@ +--- +dnbd3_is_proxy: true +dnbd3_base_path: "/mnt/dnbd3" diff --git a/hosts b/hosts index 83b2a1ed1..9233eb2d2 100644 --- a/hosts +++ b/hosts @@ -94,3 +94,13 @@ ansible_group_priority=4 [dokku] apps.galaxyproject.eu ansible_ssh_user=ubuntu + +[dnbd3:children] +dnbd3-primary +dnbd3-proxy + +[dnbd3_primary] +dnbd3-primary.galaxyproject.eu + +[dnbd3_proxy] +dnbd3-proxy.galaxyproject.eu ansible_ssh_user=ubuntu diff --git a/requirements.yaml b/requirements.yaml index 293c1766e..5902750f3 100644 --- a/requirements.yaml +++ b/requirements.yaml @@ -157,3 +157,11 @@ roles: - src: https://github.com/usegalaxy-eu/ansible-fw-glxeu-generic name: usegalaxy_eu.firewall version: 1.0.1 + - name: usegalaxy_eu.dndb3 + src: https://github.com/usegalaxy-eu/ansible-dnbd3 + version: main + - name: usegalaxy_eu.tftp + src: https://github.com/usegalaxy-eu/ansible-tftp + version: main + - name: geerlingguy.apache + version: 4.0.0 From 2c47c7237b2ce8d43c71dcf31c228587fd1f027b Mon Sep 17 00:00:00 2001 From: Mira Kuntz Date: Wed, 8 Jan 2025 17:17:29 +0100 Subject: [PATCH 07/62] dndb3 primary ssh user --- hosts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/hosts b/hosts index 9233eb2d2..abcff2434 100644 --- a/hosts +++ b/hosts @@ -100,7 +100,7 @@ dnbd3-primary dnbd3-proxy [dnbd3_primary] -dnbd3-primary.galaxyproject.eu +dnbd3-primary.galaxyproject.eu ansible_ssh_user=root [dnbd3_proxy] -dnbd3-proxy.galaxyproject.eu ansible_ssh_user=ubuntu +dnbd3-proxy.galaxyproject.eu From fc31485f1f63e9b8a597efd5ae94a3c877e9bd5d Mon Sep 17 00:00:00 2001 From: Mira Kuntz Date: Wed, 8 Jan 2025 17:58:08 +0100 Subject: [PATCH 08/62] more dndb typos --- dnbd3.yml | 10 +++++----- requirements.yaml | 2 +- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/dnbd3.yml b/dnbd3.yml index 3bf98ddc4..6167dfe20 100644 --- a/dnbd3.yml +++ b/dnbd3.yml @@ -28,7 +28,7 @@ path: "{{ tftpboot_path }}" owner: "{{ dnbd3_user }}" group: "{{ dnbd3_group }}" - when: "'dndb3_primary' in group_names" + when: "'dnbd3_primary' in group_names" roles: # - role: usegalaxy_eu.handy.os_setup # become: true @@ -36,12 +36,12 @@ # hostname: "{{ inventory_hostname }}" # enable_hostname: true # enable_powertools: true - # when: "'dndb3_primary' in group_names" + # when: "'dnbd3_primary' in group_names" - role: usegalaxy_eu.dnbd3 - role: usegalaxy_eu.tftp - when: "'dndb3_primary' in group_names" + when: "'dnbd3_primary' in group_names" - role: geerlingguy.apache - when: "'dndb3_primary' in group_names" + when: "'dnbd3_primary' in group_names" post_tasks: @@ -52,4 +52,4 @@ state: directory owner: "www-data" group: "www-data" - when: "'dndb3_primary' in group_names" + when: "'dnbd3_primary' in group_names" diff --git a/requirements.yaml b/requirements.yaml index 5902750f3..67d0f1bd6 100644 --- a/requirements.yaml +++ b/requirements.yaml @@ -157,7 +157,7 @@ roles: - src: https://github.com/usegalaxy-eu/ansible-fw-glxeu-generic name: usegalaxy_eu.firewall version: 1.0.1 - - name: usegalaxy_eu.dndb3 + - name: usegalaxy_eu.dnbd3 src: https://github.com/usegalaxy-eu/ansible-dnbd3 version: main - name: usegalaxy_eu.tftp From b320cb7eb43fdf8b82a0fa8bbc70f4a0386e5c0c Mon Sep 17 00:00:00 2001 From: Mira Kuntz Date: Wed, 8 Jan 2025 18:05:16 +0100 Subject: [PATCH 09/62] fix hosts --- dnbd3.yml | 4 +--- hosts | 4 ++-- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/dnbd3.yml b/dnbd3.yml index 6167dfe20..874fb79bb 100644 --- a/dnbd3.yml +++ b/dnbd3.yml @@ -1,7 +1,5 @@ --- -- hosts: - - dnbd3-primary - - dnbd3-proxy +- hosts: dnbd3-primary,dnbd3-proxy become: true pre_tasks: - name: Create dnbd3 user diff --git a/hosts b/hosts index abcff2434..96a074438 100644 --- a/hosts +++ b/hosts @@ -99,8 +99,8 @@ apps.galaxyproject.eu ansible_ssh_user=ubuntu dnbd3-primary dnbd3-proxy -[dnbd3_primary] +[dnbd3-primary] dnbd3-primary.galaxyproject.eu ansible_ssh_user=root -[dnbd3_proxy] +[dnbd3-proxy] dnbd3-proxy.galaxyproject.eu From 91af54dd0d7fc920d96efbb25ddb0f4820597cf2 Mon Sep 17 00:00:00 2001 From: Mira Kuntz Date: Wed, 8 Jan 2025 18:10:17 +0100 Subject: [PATCH 10/62] add git --- dnbd3.yml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/dnbd3.yml b/dnbd3.yml index 874fb79bb..7c2f69200 100644 --- a/dnbd3.yml +++ b/dnbd3.yml @@ -2,6 +2,12 @@ - hosts: dnbd3-primary,dnbd3-proxy become: true pre_tasks: + - name: Install dependencies + ansible.builtin.package: + name: + - git + state: present + - name: Create dnbd3 user ansible.builtin.group: name: "{{ dnbd3_group }}" From 92332548ee17f4f94443c6c3e2c34a3d956d3c14 Mon Sep 17 00:00:00 2001 From: Mira Kuntz Date: Wed, 15 Jan 2025 16:35:00 +0100 Subject: [PATCH 11/62] dndb3 tftp and http running on debian/zfs0f --- dnbd3.yml | 19 +++++++------- group_vars/dnbd3/vars.yml | 26 ++----------------- .../{dnbd3_primary => dnbd3primary}/vars.yml | 4 ++- .../{dnbd3_proxy => dnbd3proxy}/vars.yml | 0 hosts | 8 +++--- 5 files changed, 19 insertions(+), 38 deletions(-) rename group_vars/{dnbd3_primary => dnbd3primary}/vars.yml (51%) rename group_vars/{dnbd3_proxy => dnbd3proxy}/vars.yml (100%) diff --git a/dnbd3.yml b/dnbd3.yml index 7c2f69200..c94f4ca90 100644 --- a/dnbd3.yml +++ b/dnbd3.yml @@ -1,5 +1,6 @@ --- -- hosts: dnbd3-primary,dnbd3-proxy +- name: DNBD3 Infrastructure + hosts: dnbd3primary,dnbd3proxy become: true pre_tasks: - name: Install dependencies @@ -25,14 +26,14 @@ state: directory owner: "{{ dnbd3_user }}" group: "{{ dnbd3_group }}" - - name: Grant access to http directory + - name: Grant access to tftp root directory ansible.builtin.file: mode: "770" - state: directory path: "{{ tftpboot_path }}" - owner: "{{ dnbd3_user }}" - group: "{{ dnbd3_group }}" - when: "'dnbd3_primary' in group_names" + state: directory + owner: "{{ tftp_user }}" + group: "{{ tftp_group }}" + when: "'dnbd3primary' in group_names" roles: # - role: usegalaxy_eu.handy.os_setup # become: true @@ -43,9 +44,9 @@ # when: "'dnbd3_primary' in group_names" - role: usegalaxy_eu.dnbd3 - role: usegalaxy_eu.tftp - when: "'dnbd3_primary' in group_names" + when: "'dnbd3primary' in group_names" - role: geerlingguy.apache - when: "'dnbd3_primary' in group_names" + when: "'dnbd3primary' in group_names" post_tasks: @@ -56,4 +57,4 @@ state: directory owner: "www-data" group: "www-data" - when: "'dnbd3_primary' in group_names" + when: "'dnbd3primary' in group_names" diff --git a/group_vars/dnbd3/vars.yml b/group_vars/dnbd3/vars.yml index 708c9fba1..b01be7d0a 100644 --- a/group_vars/dnbd3/vars.yml +++ b/group_vars/dnbd3/vars.yml @@ -1,37 +1,15 @@ --- # --- dnbd3 setup vars --- #dnbd3_base_path: "/mnt/s3/dnbd3" -dnbd3_log_file: "/tmp/dnbd3-server.log" -dnbd3_config_path: "/usr/local/etc/dnbd3-server" -dnbd3_user: "dnbd3" -dnbd3_group: "dnbd3" -dnbd3_server_conf: "{{ dnbd3_base_path }}/server.conf" -dnbd3_alt_servers_conf: "{{ dnbd3_base_path }}/alt-servers" -# --- dnbd3 repo vars --- dnbd3_git_tag: "master" dnbd3_git_repo: "git://git.openslx.org/dnbd3.git" # --- dnbd3 config vars --- dnbd3_listen_port: "5003" dnbd3_client_penalty: "100000" -dnbd3_primary: "{{ hostvars[dnbd3_primary]['ansible_default_ipv4']['address'] }}" +dnbd3_primary: "{{ hostvars['dnbd3-primary.galaxyproject.eu']['ansible_default_ipv4']['address'] }}" dnbd3_primary_comment: "Primary DNBD3 server, which replicates S3FS mounted QCOW2 images." # dnbd3_primary_backup: "10.20.56.174" # dnbd3_primary_backup_comment: "Secondary/backup DNBD3 server, which replicates S3FS mounted QCOW2 images." dnbd3_poxys: - - ip: "{{ hostvars[dnbd3_proxy]['ansible_default_ipv4']['address'] }}" + - ip: "{{ hostvars[dnbd3-proxy]['ansible_default_ipv4']['address'] }}" comment: "1st DNBD3 Proxy" - - -# --- dnbd3 build vars --- -dnbd3_src_dir: "/usr/local/src/dnbd3" -dnbd3_build_dir: "{{ dnbd3_src_dir }}/build" -dnbd3_server_binary: "/usr/local/sbin/dnbd3-server" -dnbd3_new_binary: "{{ dnbd3_src_dir }}/dnbd3-server.new" -dnbd3_old_binary: "{{ dnbd3_src_dir }}/dnbd3-server.old" - -# --- dnbd3 systemd vars --- -dnbd3_systemd_dir: "/etc/systemd/system" -dnbd3_service_dir: "{{ dnbd3_systemd_dir }}/dnbd3-server.service.d" -dnbd3_service_file: "{{ dnbd3_systemd_dir }}/dnbd3-server.service" -dnbd3_limits_file: "{{ dnbd3_service_dir }}/limits.conf" -dnbd3_srv_mount_file: "{{ dnbd3_systemd_dir }}/srv.mount" diff --git a/group_vars/dnbd3_primary/vars.yml b/group_vars/dnbd3primary/vars.yml similarity index 51% rename from group_vars/dnbd3_primary/vars.yml rename to group_vars/dnbd3primary/vars.yml index 81ec927bf..16e9f1a93 100644 --- a/group_vars/dnbd3_primary/vars.yml +++ b/group_vars/dnbd3primary/vars.yml @@ -3,6 +3,8 @@ dnbd3_is_proxy: false dnbd3_base_path: "/export/dnbd3-cache/images" tftpboot_path: "/export/dnbd3-cache/boot" apache_listen_port: 80 +apache_remove_default_vhost: true apache_vhosts: - - servername: "example.com" + - servername: "dnbd3-primary.galaxyproject.eu" + serveralias: "{{ hostvars['dnbd3-primary.galaxyproject.eu']['ansible_default_ipv4']['address'] }}" documentroot: "/export/dnbd3-cache/http" diff --git a/group_vars/dnbd3_proxy/vars.yml b/group_vars/dnbd3proxy/vars.yml similarity index 100% rename from group_vars/dnbd3_proxy/vars.yml rename to group_vars/dnbd3proxy/vars.yml diff --git a/hosts b/hosts index 96a074438..b5d7c8e13 100644 --- a/hosts +++ b/hosts @@ -96,11 +96,11 @@ ansible_group_priority=4 apps.galaxyproject.eu ansible_ssh_user=ubuntu [dnbd3:children] -dnbd3-primary -dnbd3-proxy +dnbd3primary +dnbd3proxy -[dnbd3-primary] +[dnbd3primary] dnbd3-primary.galaxyproject.eu ansible_ssh_user=root -[dnbd3-proxy] +[dnbd3proxy] dnbd3-proxy.galaxyproject.eu From efd20c8481b37ba8771b457bdc3d0eb1a9b03bd3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jos=C3=A9=20Manuel=20Dom=C3=ADnguez?= Date: Mon, 3 Feb 2025 10:10:24 +0100 Subject: [PATCH 12/62] Allow Galaxy headnode to connect to 'gxitproxy' database as user 'galaxy' A separate database for the Galaxy Interactive Tools Proxy is needed to store the interactive tools session map on PostgreSQL. @jdavcs wrote (https://github.com/galaxyproject/galaxy/pull/18481#issuecomment-2218493956): > I assume the gxitproxy table is meant to be part of the main galaxy's database (as per https://github.com/usegalaxy-eu/infrastructure-playbook/pull/1251)? If so, that may break things. Galaxy does not assume the existence of tables in its database that are not defined in its model (except for this one case). Thus, db schema migration logic may break, now or in the future when we add some feature that scans the database). This commit is part of the migration procedure to store interactive tool sessions in PostgreSQL database 'gxitproxy' (https://github.com/usegalaxy-eu/infrastructure-playbook/pull/1251). In addition to merging it into `master`, please complete step (1) of the migration (read below). > On Postgres, create the database gxitproxy and set the galaxy user as its owner. Create also the table gxitproxy on beforehand (gx-it-proxy refuses to run without it). For greater details you may have a look at the updated GTN training (https://github.com/usegalaxy-eu/infrastructure-playbook/pull/galaxyproject/training-material/pull/5179). --- group_vars/sn05.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/group_vars/sn05.yml b/group_vars/sn05.yml index c7404438c..c65a49582 100644 --- a/group_vars/sn05.yml +++ b/group_vars/sn05.yml @@ -88,6 +88,7 @@ postgresql_pg_hba_conf: - "host tiaas tiaas 132.230.223.239/32 md5" - "host tiaas tiaas 132.230.223.238/32 md5" - "host tiaas tiaas 10.5.68.237/32 md5" + - "host gxitproxy galaxy 132.230.223.239/32 md5" - "host galaxy-test galaxy-test 132.230.223.239/32 md5" - "host galaxy-test galaxy-test 10.5.68.0/24 md5" - "host galaxy galaxyftp 132.230.224.107/32 md5" From 5adf4dcef6c11d653afb58ce31b0e85dc1603302 Mon Sep 17 00:00:00 2001 From: Sanjay Kumar Srikakulam Date: Wed, 5 Feb 2025 15:20:30 +0000 Subject: [PATCH 13/62] update galaxy_jwd script to skip if the backend is not of type disk and if backend is removed from the object_store_conf file --- roles/usegalaxy-eu.bashrc/files/galaxy_jwd.py | 23 ++++++++++++------- 1 file changed, 15 insertions(+), 8 deletions(-) diff --git a/roles/usegalaxy-eu.bashrc/files/galaxy_jwd.py b/roles/usegalaxy-eu.bashrc/files/galaxy_jwd.py index 4964121ff..3f1830274 100644 --- a/roles/usegalaxy-eu.bashrc/files/galaxy_jwd.py +++ b/roles/usegalaxy-eu.bashrc/files/galaxy_jwd.py @@ -301,11 +301,13 @@ def parse_object_store(object_store_conf: str) -> dict: backends = {} for backend in dom.getElementsByTagName("backend"): backend_id = backend.getAttribute("id") - backends[backend_id] = {} + backend_type = backend.getAttribute("type") + backends[backend_id] = {"type": backend_type, "job_work_path": None} + # Get the extra_dir's path for each backend if type is "job_work" for extra_dir in backend.getElementsByTagName("extra_dir"): if extra_dir.getAttribute("type") == "job_work": - backends[backend_id] = extra_dir.getAttribute("path") + backends[backend_id]["job_work_path"] = extra_dir.getAttribute("path") return backends @@ -346,20 +348,25 @@ def decode_path( Args: job_id: Job id. metadata: List of object_store_id and update_time. - backends_dict: Dictionary of backend id and path of type 'job_work'. + backends_dict: Dictionary of backend id with its type and path of type 'job_work'. job_runner_name: Name of the job runner. Defaults to None. Returns: Path to the JWD. """ job_id = str(job_id) + object_store_id = metadata[0] + backend_info = backends_dict.get(object_store_id) # Check if object_store_id exists in our object store config - if metadata[0] not in backends_dict.keys(): - raise ValueError( - f"Object store id '{metadata[0]}' does not exist in the " - f"object_store_conf.xml file." - ) + if not backend_info: + print(f"Skipping backend '{object_store_id}' as it does not exist in the object_store_conf.xml file.") + return None + + # Check if the backend type is 'disk' + if backend_info["type"] != "disk": + print(f"Skipping backend '{object_store_id}' as its type is '{backend_info['type']}' and not 'disk'.") + return None # Pulsar embedded jobs uses the staging directory and this has a different # path structure From 5c65210082de5eae94f118d3cb87a061519731e9 Mon Sep 17 00:00:00 2001 From: Sanjay Kumar Srikakulam Date: Wed, 5 Feb 2025 15:35:58 +0000 Subject: [PATCH 14/62] Update the jwd_path decode var --- roles/usegalaxy-eu.bashrc/files/galaxy_jwd.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roles/usegalaxy-eu.bashrc/files/galaxy_jwd.py b/roles/usegalaxy-eu.bashrc/files/galaxy_jwd.py index 3f1830274..9a53f9f76 100644 --- a/roles/usegalaxy-eu.bashrc/files/galaxy_jwd.py +++ b/roles/usegalaxy-eu.bashrc/files/galaxy_jwd.py @@ -374,7 +374,7 @@ def decode_path( jwd_path = f"{backends_dict['pulsar_embedded']}/{job_id}" else: jwd_path = ( - f"{backends_dict[metadata[0]]}/" + f"{backend_info['job_work_path']}/" f"0{job_id[0:2]}/{job_id[2:5]}/{job_id}" ) From 77706d39b43194543cd0faf1a55355ebdb3f38e6 Mon Sep 17 00:00:00 2001 From: Sanjay Kumar Srikakulam Date: Wed, 5 Feb 2025 15:44:36 +0000 Subject: [PATCH 15/62] python lint fix --- roles/usegalaxy-eu.bashrc/files/galaxy_jwd.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/roles/usegalaxy-eu.bashrc/files/galaxy_jwd.py b/roles/usegalaxy-eu.bashrc/files/galaxy_jwd.py index 9a53f9f76..c56711fcc 100644 --- a/roles/usegalaxy-eu.bashrc/files/galaxy_jwd.py +++ b/roles/usegalaxy-eu.bashrc/files/galaxy_jwd.py @@ -360,12 +360,14 @@ def decode_path( # Check if object_store_id exists in our object store config if not backend_info: - print(f"Skipping backend '{object_store_id}' as it does not exist in the object_store_conf.xml file.") + print(f"Skipping backend '{object_store_id}' as it does not exist in" + f" the object_store_conf.xml file.") return None # Check if the backend type is 'disk' if backend_info["type"] != "disk": - print(f"Skipping backend '{object_store_id}' as its type is '{backend_info['type']}' and not 'disk'.") + print(f"Skipping backend '{object_store_id}' as its type " + f"is '{backend_info['type']}' and not 'disk'.") return None # Pulsar embedded jobs uses the staging directory and this has a different From 69a521c2f8f44da5de124fb3fbf84836d33761b5 Mon Sep 17 00:00:00 2001 From: Sanjay Kumar Srikakulam Date: Wed, 5 Feb 2025 15:49:35 +0000 Subject: [PATCH 16/62] Python flake8 lint fix --- roles/usegalaxy-eu.bashrc/files/galaxy_jwd.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/roles/usegalaxy-eu.bashrc/files/galaxy_jwd.py b/roles/usegalaxy-eu.bashrc/files/galaxy_jwd.py index c56711fcc..878ffbdc3 100644 --- a/roles/usegalaxy-eu.bashrc/files/galaxy_jwd.py +++ b/roles/usegalaxy-eu.bashrc/files/galaxy_jwd.py @@ -307,7 +307,8 @@ def parse_object_store(object_store_conf: str) -> dict: # Get the extra_dir's path for each backend if type is "job_work" for extra_dir in backend.getElementsByTagName("extra_dir"): if extra_dir.getAttribute("type") == "job_work": - backends[backend_id]["job_work_path"] = extra_dir.getAttribute("path") + backends[backend_id]["job_work_path"] = \ + extra_dir.getAttribute("path") return backends @@ -348,7 +349,8 @@ def decode_path( Args: job_id: Job id. metadata: List of object_store_id and update_time. - backends_dict: Dictionary of backend id with its type and path of type 'job_work'. + backends_dict: Dictionary of backend id with its type and path of type + 'job_work'. job_runner_name: Name of the job runner. Defaults to None. Returns: From a6450ab4a0375d94e0d9136343cfb9233204802a Mon Sep 17 00:00:00 2001 From: Sanjay Kumar Srikakulam Date: Thu, 6 Feb 2025 08:45:00 +0000 Subject: [PATCH 17/62] remove disk type based conditional check when decoding path user object stores will anyway get filtered out when checking whether the object_store_id exists in the object_store_conf.xml. So, the error would not appear. --- roles/usegalaxy-eu.bashrc/files/galaxy_jwd.py | 6 ------ 1 file changed, 6 deletions(-) diff --git a/roles/usegalaxy-eu.bashrc/files/galaxy_jwd.py b/roles/usegalaxy-eu.bashrc/files/galaxy_jwd.py index 878ffbdc3..ad7a40d92 100644 --- a/roles/usegalaxy-eu.bashrc/files/galaxy_jwd.py +++ b/roles/usegalaxy-eu.bashrc/files/galaxy_jwd.py @@ -366,12 +366,6 @@ def decode_path( f" the object_store_conf.xml file.") return None - # Check if the backend type is 'disk' - if backend_info["type"] != "disk": - print(f"Skipping backend '{object_store_id}' as its type " - f"is '{backend_info['type']}' and not 'disk'.") - return None - # Pulsar embedded jobs uses the staging directory and this has a different # path structure if (job_runner_name or "").startswith("pulsar_embedded"): From 5ad4bfaa67df30145cab6b96ae8c6bb8bab52f6f Mon Sep 17 00:00:00 2001 From: Bjoern Gruening Date: Sat, 8 Feb 2025 22:39:59 +0100 Subject: [PATCH 18/62] add new file sources --- templates/galaxy/config/file_source_templates.yml.j2 | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/templates/galaxy/config/file_source_templates.yml.j2 b/templates/galaxy/config/file_source_templates.yml.j2 index 071bd4a6d..4a18ee851 100644 --- a/templates/galaxy/config/file_source_templates.yml.j2 +++ b/templates/galaxy/config/file_source_templates.yml.j2 @@ -1,6 +1,11 @@ # This is a catalog file for all the user file source templates that are offered by EU's Galaxy server. +- include: "{{ galaxy_server_dir }}/lib/galaxy/files/templates/examples/onedata.yml" +- include: "{{ galaxy_server_dir }}/lib/galaxy/files/templates/examples/production_aws_private_bucket.yml" +- include: "{{ galaxy_server_dir }}/lib/galaxy/files/templates/examples/production_aws_public_bucket.yml" - include: "{{ galaxy_server_dir }}/lib/galaxy/files/templates/examples/production_azure.yml" +- include: "{{ galaxy_server_dir }}/lib/galaxy/files/templates/examples/production_dropbox.yml" - include: "{{ galaxy_server_dir }}/lib/galaxy/files/templates/examples/production_ftp.yml" +- include: "{{ galaxy_server_dir }}/lib/galaxy/files/templates/examples/production_google_drive.yml" - include: "{{ galaxy_server_dir }}/lib/galaxy/files/templates/examples/production_s3fs.yml" -- include: "{{ galaxy_server_dir }}/lib/galaxy/files/templates/examples/production_aws_private_bucket.yml" -- include: "{{ galaxy_server_dir }}/lib/galaxy/files/templates/examples/production_aws_public_bucket.yml" +- include: "{{ galaxy_server_dir }}/lib/galaxy/files/templates/examples/production_webdav.yml" + From 58dcaa80301f3c008531680327ec63972bc68626 Mon Sep 17 00:00:00 2001 From: Bjoern Gruening Date: Sat, 8 Feb 2025 22:53:55 +0100 Subject: [PATCH 19/62] add new object stores --- templates/galaxy/config/object_store_templates.yml.j2 | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/templates/galaxy/config/object_store_templates.yml.j2 b/templates/galaxy/config/object_store_templates.yml.j2 index 8d0c46b76..110e0e0f3 100644 --- a/templates/galaxy/config/object_store_templates.yml.j2 +++ b/templates/galaxy/config/object_store_templates.yml.j2 @@ -1,5 +1,7 @@ # This is a catalog file for all the user object store templates that are offered by EU's Galaxy server. -- include: "{{ galaxy_server_dir }}/lib/galaxy/objectstore/templates/examples/production_azure_blob.yml" +- include: "{{ galaxy_server_dir }}/lib/galaxy/objectstore/templates/examples/onedata.yml" - include: "{{ galaxy_server_dir }}/lib/galaxy/objectstore/templates/examples/production_aws_s3.yml" -- include: "{{ galaxy_server_dir }}/lib/galaxy/objectstore/templates/examples/production_generic_s3.yml" +- include: "{{ galaxy_server_dir }}/lib/galaxy/objectstore/templates/examples/production_azure_blob.yml" - include: "{{ galaxy_server_dir }}/lib/galaxy/objectstore/templates/examples/production_gcp_s3.yml" +- include: "{{ galaxy_server_dir }}/lib/galaxy/objectstore/templates/examples/production_generic_s3.yml" + From efe9247a36cf2cde7aea07e16d45c7eefab6cb01 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B6rn=20Gr=C3=BCning?= Date: Sun, 9 Feb 2025 01:07:11 +0100 Subject: [PATCH 20/62] update entry --- files/galaxy/tpv/tools.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/files/galaxy/tpv/tools.yml b/files/galaxy/tpv/tools.yml index b9879a378..7290a9f07 100644 --- a/files/galaxy/tpv/tools.yml +++ b/files/galaxy/tpv/tools.yml @@ -218,7 +218,7 @@ tools: if: | helpers.job_args_match(job, app, {"mode": {"mode_selector": "screen"}}) mem: 480 - cores: 24 + cores: 64 toolshed.g2.bx.psu.edu/repos/computational-metabolomics/sirius_csifingerid/sirius_csifingerid/.*: cores: 8 From 03fdd5a7c6b4b945cf9c7f0df9b3c942da54fbc6 Mon Sep 17 00:00:00 2001 From: Sanjay Kumar Srikakulam Date: Mon, 10 Feb 2025 13:49:51 +0000 Subject: [PATCH 21/62] Add a TPV climate strike rule to defer jobs during the climate strike --- files/galaxy/tpv/tool_defaults.yml | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/files/galaxy/tpv/tool_defaults.yml b/files/galaxy/tpv/tool_defaults.yml index 87e33030a..48562e6cb 100644 --- a/files/galaxy/tpv/tool_defaults.yml +++ b/files/galaxy/tpv/tool_defaults.yml @@ -65,6 +65,28 @@ tools: retval fail: | Invalid 'Remote resources id' selected in the config menu under 'User -> Preferences -> Manage Information -> Use distributed compute resources'. Please reselect either 'default' or an appropriate remote resource then click 'Save' and rerun your job. + - id: climate_strike + # delay jobs using HTCondor's job deferral feature + # https://htcondor.readthedocs.io/en/latest/users-manual/time-scheduling-for-job-execution.html + if: True + execute: | + from datetime import datetime + + strike_start = datetime(2025,2,14,7,0) + strike_end = datetime(2023,2,14,20,0) + + training_roles = ( + [r.name for r in user.all_roles() + if not r.deleted and r.name in + ("training-master2025", "training-metap-intro")] + if user is not None else [] + ) + + now = datetime.now() + if strike_start <= now < strike_end and not training_roles: + entity.params["deferral_time"] = f"{int(strike_end.timestamp())}" + entity.params["deferral_prep_time"] = "60" + entity.params["deferral_window"] = "864000" # 10 days rank: | final_destinations = helpers.weighted_random_sampling(candidate_destinations) final_destinations From 450061ca4d7c4eb3c0e2d2cf55cc0502cf9cd168 Mon Sep 17 00:00:00 2001 From: Sanjay Kumar Srikakulam Date: Mon, 10 Feb 2025 13:54:12 +0000 Subject: [PATCH 22/62] correct the strike end timestamp --- files/galaxy/tpv/tool_defaults.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/files/galaxy/tpv/tool_defaults.yml b/files/galaxy/tpv/tool_defaults.yml index 48562e6cb..a9f7be4a6 100644 --- a/files/galaxy/tpv/tool_defaults.yml +++ b/files/galaxy/tpv/tool_defaults.yml @@ -73,7 +73,7 @@ tools: from datetime import datetime strike_start = datetime(2025,2,14,7,0) - strike_end = datetime(2023,2,14,20,0) + strike_end = datetime(2025,2,14,20,0) training_roles = ( [r.name for r in user.all_roles() From 1d9bba6b3f16949270c260b82c341104a146e901 Mon Sep 17 00:00:00 2001 From: Sanjay Kumar Srikakulam Date: Mon, 10 Feb 2025 14:01:18 +0000 Subject: [PATCH 23/62] fix indentation --- files/galaxy/tpv/tool_defaults.yml | 28 ++++++++++++++-------------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/files/galaxy/tpv/tool_defaults.yml b/files/galaxy/tpv/tool_defaults.yml index a9f7be4a6..5f1091c3f 100644 --- a/files/galaxy/tpv/tool_defaults.yml +++ b/files/galaxy/tpv/tool_defaults.yml @@ -70,23 +70,23 @@ tools: # https://htcondor.readthedocs.io/en/latest/users-manual/time-scheduling-for-job-execution.html if: True execute: | - from datetime import datetime + from datetime import datetime - strike_start = datetime(2025,2,14,7,0) - strike_end = datetime(2025,2,14,20,0) + strike_start = datetime(2025,2,14,7,0) + strike_end = datetime(2025,2,14,20,0) - training_roles = ( - [r.name for r in user.all_roles() - if not r.deleted and r.name in - ("training-master2025", "training-metap-intro")] - if user is not None else [] - ) + training_roles = ( + [r.name for r in user.all_roles() + if not r.deleted and r.name in + ("training-master2025", "training-metap-intro")] + if user is not None else [] + ) - now = datetime.now() - if strike_start <= now < strike_end and not training_roles: - entity.params["deferral_time"] = f"{int(strike_end.timestamp())}" - entity.params["deferral_prep_time"] = "60" - entity.params["deferral_window"] = "864000" # 10 days + now = datetime.now() + if strike_start <= now < strike_end and not training_roles: + entity.params["deferral_time"] = f"{int(strike_end.timestamp())}" + entity.params["deferral_prep_time"] = "60" + entity.params["deferral_window"] = "864000" # 10 days rank: | final_destinations = helpers.weighted_random_sampling(candidate_destinations) final_destinations From b85d8eaa8db640e3bb1003b65e78ed786f8eb091 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B6rn=20Gr=C3=BCning?= Date: Mon, 10 Feb 2025 16:08:21 +0100 Subject: [PATCH 24/62] set file_source_temp_dir --- group_vars/gxconfig.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/group_vars/gxconfig.yml b/group_vars/gxconfig.yml index a93cf1fa1..ef2792725 100644 --- a/group_vars/gxconfig.yml +++ b/group_vars/gxconfig.yml @@ -2707,7 +2707,7 @@ base_app_main: &BASE_APP_MAIN # Directory to store temporary files for file sources. This defaults # to new_file_path if not set. - #file_source_temp_dir: null + file_source_temp_dir: /data/jwd04/tmp/ # Default value for use_temp_files for webdav plugins that don't # explicitly declare this. From ccd3cdc0836049eae23a50c1e9189c092c54c141 Mon Sep 17 00:00:00 2001 From: Amirhossein Naghsh Nilchi Date: Tue, 11 Feb 2025 10:57:24 +0100 Subject: [PATCH 25/62] add cellxgeneVIP --- files/galaxy/tpv/interactive_tools.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/files/galaxy/tpv/interactive_tools.yml b/files/galaxy/tpv/interactive_tools.yml index 60cf91ce8..18a4ad565 100644 --- a/files/galaxy/tpv/interactive_tools.yml +++ b/files/galaxy/tpv/interactive_tools.yml @@ -214,6 +214,10 @@ tools: inherits: interactive_tool cores: 1 mem: 4 + interactive_tool_cellxgene_vip: + inherits: interactive_tool + cores: 1 + mem: 4 interactive_tool_bam_iobio: inherits: interactive_tool cores: 1 From cd4db884f92c0110ba5dd9497c4ab1b35d75bc89 Mon Sep 17 00:00:00 2001 From: Amirhossein Naghsh Nilchi Date: Tue, 11 Feb 2025 11:19:46 +0100 Subject: [PATCH 26/62] add config file --- templates/galaxy/config/tool_conf.xml.j2 | 1 + 1 file changed, 1 insertion(+) diff --git a/templates/galaxy/config/tool_conf.xml.j2 b/templates/galaxy/config/tool_conf.xml.j2 index 1449b4591..06e3639a8 100644 --- a/templates/galaxy/config/tool_conf.xml.j2 +++ b/templates/galaxy/config/tool_conf.xml.j2 @@ -392,6 +392,7 @@ +