From e295ddd21477c9fd716ed09cd776a48439e438ac Mon Sep 17 00:00:00 2001 From: Michael Schubert Date: Fri, 2 Aug 2024 13:51:02 +0200 Subject: [PATCH] minor doc --- R/Q.r | 8 +++---- _pkgdown.yml | 1 + man/Q.Rd | 8 +++---- man/Q_rows.Rd | 8 +++---- man/host.Rd | 2 +- vignettes/faq.Rmd | 28 ++++++++++++++++--------- vignettes/userguide.Rmd | 46 ++++++++++++++++++++++++++++++----------- 7 files changed, 66 insertions(+), 35 deletions(-) diff --git a/R/Q.r b/R/Q.r index ad23661..7329d9e 100644 --- a/R/Q.r +++ b/R/Q.r @@ -6,10 +6,10 @@ #' @param export List of objects to be exported to the worker #' @param pkgs Character vector of packages to load on the worker #' @param seed A seed to set for each function call -#' @param memory Short for template=list(memory=value) -#' @param template A named list of values to fill in template -#' @param n_jobs The number of LSF jobs to submit; upper limit of jobs -#' if job_size is given as well +#' @param memory Short for `template=list(memory=value)` +#' @param template A named list of values to fill in the scheduler template +#' @param n_jobs The number of jobs to submit; upper limit of jobs if job_size +#' is given as well #' @param job_size The number of function calls per job #' @param split_array_by The dimension number to split any arrays in `...`; default: last #' @param rettype Return type of function call (vector type or 'list') diff --git a/_pkgdown.yml b/_pkgdown.yml index 995f91e..1fec95d 100644 --- a/_pkgdown.yml +++ b/_pkgdown.yml @@ -1,5 +1,6 @@ template: bootstrap: 5 + light-switch: true toc: depth: 2 # level 3 currently not distinguishable from 2 diff --git a/man/Q.Rd b/man/Q.Rd index b193795..d7fbf38 100644 --- a/man/Q.Rd +++ b/man/Q.Rd @@ -39,12 +39,12 @@ Q( \item{seed}{A seed to set for each function call} -\item{memory}{Short for template=list(memory=value)} +\item{memory}{Short for `template=list(memory=value)`} -\item{template}{A named list of values to fill in template} +\item{template}{A named list of values to fill in the scheduler template} -\item{n_jobs}{The number of LSF jobs to submit; upper limit of jobs -if job_size is given as well} +\item{n_jobs}{The number of jobs to submit; upper limit of jobs if job_size +is given as well} \item{job_size}{The number of function calls per job} diff --git a/man/Q_rows.Rd b/man/Q_rows.Rd index 2d38d83..05b88b3 100644 --- a/man/Q_rows.Rd +++ b/man/Q_rows.Rd @@ -38,12 +38,12 @@ Q_rows( \item{seed}{A seed to set for each function call} -\item{memory}{Short for template=list(memory=value)} +\item{memory}{Short for `template=list(memory=value)`} -\item{template}{A named list of values to fill in template} +\item{template}{A named list of values to fill in the scheduler template} -\item{n_jobs}{The number of LSF jobs to submit; upper limit of jobs -if job_size is given as well} +\item{n_jobs}{The number of jobs to submit; upper limit of jobs if job_size +is given as well} \item{job_size}{The number of function calls per job} diff --git a/man/host.Rd b/man/host.Rd index 67fc787..0cd9c88 100644 --- a/man/host.Rd +++ b/man/host.Rd @@ -6,7 +6,7 @@ \usage{ host( node = getOption("clustermq.host", Sys.info()["nodename"]), - ports = 6000:9999, + ports = getOption("clustermq.ports", 6000:9999), n = 100 ) } diff --git a/vignettes/faq.Rmd b/vignettes/faq.Rmd index 01455b9..313e691 100644 --- a/vignettes/faq.Rmd +++ b/vignettes/faq.Rmd @@ -33,8 +33,9 @@ suppressPackageStartupMessages(library(clustermq)) To compile this package a fully C++11 compliant compiler is required. This is [implicit for CRAN packages](https://www.tidyverse.org/blog/2023/03/cran-checks-compiled-code/) -since `R=3.6.2` and is hence not listed in _SystemRequirements_. If you -encounter an error saying that that no matching function call to +since `R=3.6.2` and is hence not listed in _SystemRequirements_. + +If you encounter an error saying that that no matching function call to `zmq::message_t::message_t(std::string&)` exists, your compiler does not (fully) support this and the automated check failed for some reason. @@ -88,14 +89,21 @@ Execution halted ``` the submitted job is indeed unable to establish a network connection with the -head node. This can happen if your HPC does not allow incoming connections, but -more likely happens because there are multiple network interfaces, only some of -which have access to the head node. - -You can list the available network interfaces using the `ifconfig` command in -the terminal. Find the interface that shares a subnetwork with the head node -and add the [R option](https://mschubert.github.io/clustermq/articles/userguide.html#options) `clustermq.host=`. If this is -unclear, contact your system administrators to see which interface to use. +head node. This can happen if your HPC does not allow incoming connections at +all, but more likely happens because (1) only certain ports are allowed, or (2) +there are multiple network interfaces, only some of which have access to the +head node. + + 1. If the head node only allows incoming connections on certain ports, set the + [R + option](https://mschubert.github.io/clustermq/articles/userguide.html#options) + `clustermq.ports=`. + 2. You can list the available network interfaces using the `ifconfig` command + in the terminal. Find the interface that shares a subnetwork with the head + node and add the [R + option](https://mschubert.github.io/clustermq/articles/userguide.html#options) + `clustermq.host=`. If this is unclear, contact your system + administrators to see which interface to use. ## SSH not working {#ssh} diff --git a/vignettes/userguide.Rmd b/vignettes/userguide.Rmd index 4f4eaeb..de62968 100644 --- a/vignettes/userguide.Rmd +++ b/vignettes/userguide.Rmd @@ -37,9 +37,10 @@ the bundled library: ```{r eval=FALSE} # Recommended: -# If your system has `libzmq` installed but you want to enable the worker crash -# monitor, set the following environment variable to enable compilation of the -# bundled `libzmq` library with the required feature (`-DZMQ_BUILD_DRAFT_API=1`): +# If your system has `libzmq` installed but you want to enable the worker +# crash monitor, set the environment variable below to use the bundled +# `libzmq` library with the required feature (`-DZMQ_BUILD_DRAFT_API=1`): + # Sys.setenv(CLUSTERMQ_USE_SYSTEM_LIBZMQ=0) install.packages("clustermq") ``` @@ -237,8 +238,12 @@ we can use `clustermq`: ```{r} # set up the scheduler first, otherwise this will run sequentially -clustermq::register_dopar_cmq(n_jobs=2, memory=1024) # this accepts same arguments as `Q` -x = foreach(i=1:3) %dopar% sqrt(i) # this will be executed as jobs + +# this accepts same arguments as `Q` +clustermq::register_dopar_cmq(n_jobs=2, memory=1024) + +# this will be executed as jobs +x = foreach(i=1:3) %dopar% sqrt(i) ``` As [BiocParallel](https://bioconductor.org/packages/release/bioc/html/BiocParallel.html) @@ -413,6 +418,21 @@ options( ## Scheduler templates +The package provides its own default scheduler templates, similar to the ones +listed below. Which template is used is decided based on which scheduler +submission executable is present in the user's `$PATH`, e.g. `sbatch` for SLURM +or `bsub` for LSF. `qsub` is ambiguous between SGE and PBS/Torque, so in this +case `options(clustermq.scheduler = "")` should be set to the correct one. + +A user can provide their own template file via `options(clustermq.template = +"")`, containing arbitrary template values `{{ value | default }}`. These +values will be filled upon job submission in the following order of priority: + + 1. The argument provided to `Q(..., template=list(key = value))` or + `workers(... template=list(key = value))` + 2. The value of `getOption("clustermq.defaults")` + 3. The default value inside the template + ### LSF Set the following options in your _R_ session that will submit jobs: @@ -589,8 +609,9 @@ the missing options. Set the following options in your _R_ session that will submit jobs: ```{r eval=FALSE} -options(clustermq.scheduler = "Torque", - clustermq.template = "/path/to/file/below" # if using your own template +options( + clustermq.scheduler = "Torque", + clustermq.template = "/path/to/file/below" # if using your own template ) ``` @@ -627,11 +648,12 @@ want to use it, first make sure that `clustermq` works on your server with the real scheduler. Only then move on to setting up SSH. ```{r eval=FALSE} -options(clustermq.scheduler = "ssh", - clustermq.ssh.host = "myhost", # set this up in your local ~/.ssh/config - clustermq.ssh.log = "~/ssh_proxy.log", # log file on your HPC - clustermq.ssh.timeout = 30, # if changing the default connection timeout - clustermq.template = "/path/to/file/below" # if using your own template +options( + clustermq.scheduler = "ssh", + clustermq.ssh.host = "myhost", # set this up in your local ~/.ssh/config + clustermq.ssh.log = "~/ssh_proxy.log", # log file on your HPC + clustermq.ssh.timeout = 30, # if changing default connection timeout + clustermq.template = "/path/to/file/below" # if using your own template ) ```