Merge branch 'master' into fix_extra_tenantId_in_azure_config

nextflow-io · Mar 4, 2025 · 00567ca · 00567ca
2 parents 2697cdc + 15c6f3c
commit 00567ca
Show file tree

Hide file tree

Showing 20 changed files with 127 additions and 73 deletions.
diff --git a/build.gradle b/build.gradle
@@ -107,8 +107,8 @@ allprojects {
 
         // Documentation required libraries
         groovyDoc 'org.fusesource.jansi:jansi:2.4.0'
-        groovyDoc "org.apache.groovy:groovy-groovydoc:4.0.25"
-        groovyDoc "org.apache.groovy:groovy-ant:4.0.25"
+        groovyDoc "org.apache.groovy:groovy-groovydoc:4.0.26"
+        groovyDoc "org.apache.groovy:groovy-ant:4.0.26"
     }
 
     test {

diff --git a/docs/container.md b/docs/container.md
@@ -59,7 +59,7 @@ In the above example replace `/path/to/apptainer.img` with any Apptainer image o
 Read the {ref}`config-page` page to learn more about the `nextflow.config` file and how to use it to configure your pipeline execution.
 
 :::{note}
-Unlike Docker, Nextflow does not automatically mount host paths in the container when using Apptainer. It expects that the paths are configure and mounted system wide by the Apptainer runtime. If your Apptainer installation allows user defined bind points, read the {ref}`Apptainer configuration <config-apptainer>` section to learn how to enable Nextflow auto mounts.
+Unlike Docker, Nextflow does not automatically mount host paths in the container when using Apptainer. It expects that the paths are configured and mounted system wide by the Apptainer runtime. If your Apptainer installation allows user defined bind points, read the {ref}`Apptainer configuration <config-apptainer>` section to learn how to enable Nextflow auto mounts.
 :::
 
 :::{warning}

diff --git a/docs/process.md b/docs/process.md
@@ -473,27 +473,11 @@ workflow {
 }
 ```
 
-:::{versionadded} 23.09.0-edge
-:::
-
-By default, `path` inputs will accept any number of files and stage them accordingly. The `arity` option can be used to enforce the expected number of files, either as a number or a range.
-
-For example:
-
-```nextflow
-input:
-    path('one.txt', arity: '1')         // exactly one file is expected
-    path('pair_*.txt', arity: '2')      // exactly two files are expected
-    path('many_*.txt', arity: '1..*')   // one or more files are expected
-```
-
-When a task is executed, Nextflow will check whether the received files for each path input match the declared arity, and fail if they do not.
-
 :::{note}
 Process `path` inputs have nearly the same interface as described in {ref}`stdlib-types-path`, with one difference which is relevant when files are staged into a subdirectory. Given the following input:
 
 ```nextflow
-path x, name: 'my-dir/*'
+path x, name: 'my-dir/file.txt'
 ```
 
 In this case, `x.name` returns the file name with the parent directory (e.g. `my-dir/file.txt`), whereas normally it would return the file name (e.g. `file.txt`). You can use `x.fileName.name` to get the file name.
@@ -532,12 +516,12 @@ seq1 seq2 seq3
 
 The target input file name may contain the `*` and `?` wildcards, which can be used to control the name of staged files. The following table shows how the wildcards are replaced depending on the cardinality of the received input collection.
 
-| Cardinality | Name pattern | Staged file names                                                                                       |
+| Arity       | Name pattern | Staged file names                                                                                       |
 | ----------- | ------------ | ------------------------------------------------------------------------------------------------------- |
 | any         | `*`          | named as the source file                                                                                |
-| 1           | `file*.ext`  | `file.ext`                                                                                              |
-| 1           | `file?.ext`  | `file1.ext`                                                                                             |
-| 1           | `file??.ext` | `file01.ext`                                                                                            |
+| one         | `file*.ext`  | `file.ext`                                                                                              |
+| one         | `file?.ext`  | `file1.ext`                                                                                             |
+| one         | `file??.ext` | `file01.ext`                                                                                            |
 | many        | `file*.ext`  | `file1.ext`, `file2.ext`, `file3.ext`, ..                                                               |
 | many        | `file?.ext`  | `file1.ext`, `file2.ext`, `file3.ext`, ..                                                               |
 | many        | `file??.ext` | `file01.ext`, `file02.ext`, `file03.ext`, ..                                                            |
@@ -568,6 +552,22 @@ workflow {
 Rewriting input file names according to a named pattern is an extra feature and not at all required. The normal file input syntax introduced in the {ref}`process-input-path` section is valid for collections of multiple files as well. To handle multiple input files while preserving the original file names, use a variable identifier or the `*` wildcard.
 :::
 
+:::{versionadded} 23.09.0-edge
+:::
+
+The `arity` option can be used to enforce the expected number of files, either as a number or a range.
+
+For example:
+
+```nextflow
+input:
+path('one.txt', arity: '1')         // exactly one file is expected
+path('pair_*.txt', arity: '2')      // exactly two files are expected
+path('many_*.txt', arity: '1..*')   // one or more files are expected
+```
+
+When a task is executed, Nextflow will check whether the received files for each path input match the declared arity, and fail if they do not. When the arity is `'1'`, the corresponding input variable will be a single file; otherwise, it will be a list of files.
+
 ### Dynamic input file names
 
 When the input file name is specified by using the `name` option or a string literal, you can also use other input values as variables in the file name string. For example:
@@ -921,22 +921,6 @@ In the above example, the `randomNum` process creates a file named `result.txt`
 
 Refer to the {ref}`process reference <process-reference-outputs>` for the list of available options for `path` outputs.
 
-:::{versionadded} 23.09.0-edge
-:::
-
-By default, `path` outputs will accept any number of matching files from the task directory. The `arity` option can be used to enforce the expected number of files, either as a number or a range.
-
-For example:
-
-```nextflow
-output:
-path('one.txt', arity: '1')         // exactly one file is expected
-path('pair_*.txt', arity: '2')      // exactly two files are expected
-path('many_*.txt', arity: '1..*')   // one or more files are expected
-```
-
-When a task completes, Nextflow will check whether the produced files for each path output match the declared arity, and fail if they do not.
-
 ### Multiple output files
 
 When an output file name contains a `*` or `?` wildcard character, it is interpreted as a [glob][glob] path matcher. This allows you to capture multiple files into a list and emit the list as a single value. For example:
@@ -981,6 +965,22 @@ Although the input files matching a glob output declaration are not included in
 
 Read more about glob syntax at the following link [What is a glob?][glob]
 
+:::{versionadded} 23.09.0-edge
+:::
+
+The `arity` option can be used to enforce the expected number of files, either as a number or a range.
+
+For example:
+
+```nextflow
+output:
+path('one.txt', arity: '1')         // exactly one file is expected
+path('pair_*.txt', arity: '2')      // exactly two files are expected
+path('many_*.txt', arity: '1..*')   // one or more files are expected
+```
+
+When a task completes, Nextflow will check whether the produced files for each path output match the declared arity, and fail if they do not. When the arity is `'1'`, the corresponding output will be a single file; otherwise, it will be a list of files.
+
 ### Dynamic output file names
 
 When an output file name needs to be expressed dynamically, it is possible to define it using a dynamic string which references variables in the `input` block or in the script global context. For example:

diff --git a/docs/updating-nextflow.md b/docs/updating-nextflow.md
@@ -6,7 +6,9 @@ This page describes Nextflow release cadence, how to self-update Nextflow, and h
 
 ## Releases
 
-A stable version of Nextflow is released in the 4th and 10th month of each year. A edge version of Nextflow is released on a monthly basis. The edge version can be used to access the latest updates and experimental features. 
+A stable version of Nextflow is released in the 4th and 10th month of each year. An edge version of Nextflow is released on a monthly basis. The edge version can be used to access the latest updates and experimental features.
+
+Nextflow uses [Calendar Versioning](https://calver.org). Versions are numbered as `<year>.<month>.<patch>`. For example, `23.10.1` corresponds to the first patch of the October 2023 stable release.
 
 You can find an exhaustive list of releases and updates in the [Nextflow changelog](https://github.com/nextflow-io/nextflow/blob/master/changelog.txt).
 

diff --git a/gradle.properties b/gradle.properties
@@ -1,2 +1,3 @@
 org.gradle.caching=true
 org.gradle.jvmargs=-Xmx4g
+org.gradle.parallel=true
diff --git a/modules/nextflow/build.gradle b/modules/nextflow/build.gradle
@@ -21,12 +21,12 @@ dependencies {
     api(project(':nf-commons'))
     api(project(':nf-httpfs'))
     api 'com.github.nextflow-io.language-server:compiler:main-SNAPSHOT'
-    api "org.apache.groovy:groovy:4.0.25"
-    api "org.apache.groovy:groovy-nio:4.0.25"
-    api "org.apache.groovy:groovy-xml:4.0.25"
-    api "org.apache.groovy:groovy-json:4.0.25"
-    api "org.apache.groovy:groovy-templates:4.0.25"
-    api "org.apache.groovy:groovy-yaml:4.0.25"
+    api "org.apache.groovy:groovy:4.0.26"
+    api "org.apache.groovy:groovy-nio:4.0.26"
+    api "org.apache.groovy:groovy-xml:4.0.26"
+    api "org.apache.groovy:groovy-json:4.0.26"
+    api "org.apache.groovy:groovy-templates:4.0.26"
+    api "org.apache.groovy:groovy-yaml:4.0.26"
     api "org.slf4j:jcl-over-slf4j:2.0.16"
     api "org.slf4j:jul-to-slf4j:2.0.16"
     api "org.slf4j:log4j-over-slf4j:2.0.16"
@@ -55,7 +55,7 @@ dependencies {
     testImplementation 'org.subethamail:subethasmtp:3.1.7'
 
     // test configuration
-    testFixturesApi ("org.apache.groovy:groovy-test:4.0.25") { exclude group: 'org.apache.groovy' }
+    testFixturesApi ("org.apache.groovy:groovy-test:4.0.26") { exclude group: 'org.apache.groovy' }
     testFixturesApi ("org.objenesis:objenesis:3.4")
     testFixturesApi ("net.bytebuddy:byte-buddy:1.14.17")
     testFixturesApi ("org.spockframework:spock-core:2.3-groovy-4.0") { exclude group: 'org.apache.groovy' }

diff --git a/modules/nf-commons/build.gradle b/modules/nf-commons/build.gradle
@@ -26,8 +26,8 @@ sourceSets {
 
 dependencies {
     api "ch.qos.logback:logback-classic:1.5.16"
-    api "org.apache.groovy:groovy:4.0.25"
-    api "org.apache.groovy:groovy-nio:4.0.25"
+    api "org.apache.groovy:groovy:4.0.26"
+    api "org.apache.groovy:groovy-nio:4.0.26"
     api "commons-lang:commons-lang:2.6"
     api 'com.google.guava:guava:33.0.0-jre'
     api 'org.pf4j:pf4j:3.12.0'

diff --git a/modules/nf-httpfs/build.gradle b/modules/nf-httpfs/build.gradle
@@ -30,12 +30,12 @@ sourceSets {
 dependencies {
     api project(':nf-commons')
     api "ch.qos.logback:logback-classic:1.5.16"
-    api "org.apache.groovy:groovy:4.0.25"
-    api "org.apache.groovy:groovy-nio:4.0.25"
+    api "org.apache.groovy:groovy:4.0.26"
+    api "org.apache.groovy:groovy-nio:4.0.26"
     api("com.esotericsoftware.kryo:kryo:2.24.0") { exclude group: 'com.esotericsoftware.minlog', module: 'minlog' }
 
     /* testImplementation inherited from top gradle build file */
-    testImplementation "org.apache.groovy:groovy-json:4.0.25" // needed by wiremock
+    testImplementation "org.apache.groovy:groovy-json:4.0.26" // needed by wiremock
     testImplementation ('com.github.tomakehurst:wiremock:1.57') { exclude module: 'groovy-all' }
     testImplementation ('com.github.tomjankes:wiremock-groovy:0.2.0') { exclude module: 'groovy-all' }
 

diff --git a/plugins/nf-amazon/build.gradle b/plugins/nf-amazon/build.gradle
@@ -60,6 +60,6 @@ dependencies {
 
     testImplementation(testFixtures(project(":nextflow")))
     testImplementation project(':nextflow')
-    testImplementation "org.apache.groovy:groovy:4.0.25"
-    testImplementation "org.apache.groovy:groovy-nio:4.0.25"
+    testImplementation "org.apache.groovy:groovy:4.0.26"
+    testImplementation "org.apache.groovy:groovy-nio:4.0.26"
 }
diff --git a/plugins/nf-azure/build.gradle b/plugins/nf-azure/build.gradle
@@ -53,6 +53,6 @@ dependencies {
 
     testImplementation(testFixtures(project(":nextflow")))
     testImplementation project(':nextflow')
-    testImplementation "org.apache.groovy:groovy:4.0.25"
-    testImplementation "org.apache.groovy:groovy-nio:4.0.25"
+    testImplementation "org.apache.groovy:groovy:4.0.26"
+    testImplementation "org.apache.groovy:groovy-nio:4.0.26"
 }
diff --git a/plugins/nf-cloudcache/build.gradle b/plugins/nf-cloudcache/build.gradle
@@ -35,7 +35,7 @@ dependencies {
     compileOnly 'org.pf4j:pf4j:3.12.0'
 
     testImplementation(testFixtures(project(":nextflow")))
-    testImplementation "org.apache.groovy:groovy:4.0.25"
-    testImplementation "org.apache.groovy:groovy-nio:4.0.25"
+    testImplementation "org.apache.groovy:groovy:4.0.26"
+    testImplementation "org.apache.groovy:groovy-nio:4.0.26"
 }
 
diff --git a/plugins/nf-codecommit/build.gradle b/plugins/nf-codecommit/build.gradle
@@ -42,6 +42,6 @@ dependencies {
 
     testImplementation(testFixtures(project(":nextflow")))
     testImplementation project(':nextflow')
-    testImplementation "org.apache.groovy:groovy:4.0.25"
-    testImplementation "org.apache.groovy:groovy-nio:4.0.25"
+    testImplementation "org.apache.groovy:groovy:4.0.26"
+    testImplementation "org.apache.groovy:groovy-nio:4.0.26"
 }
diff --git a/plugins/nf-console/build.gradle b/plugins/nf-console/build.gradle
@@ -38,13 +38,13 @@ dependencies {
     compileOnly 'org.pf4j:pf4j:3.12.0'
 
     api("org.apache.groovy:groovy-console:4.0.21-patch.2") { transitive=false }
-    api("org.apache.groovy:groovy-swing:4.0.25")  { transitive=false }
+    api("org.apache.groovy:groovy-swing:4.0.26")  { transitive=false }
     // this is required by 'groovy-console'  
     api("com.github.javaparser:javaparser-core:3.25.8")
 
     testImplementation(testFixtures(project(":nextflow")))
     testImplementation project(':nextflow')
-    testImplementation "org.apache.groovy:groovy:4.0.25"
-    testImplementation "org.apache.groovy:groovy-nio:4.0.25"
+    testImplementation "org.apache.groovy:groovy:4.0.26"
+    testImplementation "org.apache.groovy:groovy-nio:4.0.26"
 }
 
diff --git a/plugins/nf-google/build.gradle b/plugins/nf-google/build.gradle
@@ -46,8 +46,8 @@ dependencies {
     api 'com.google.code.gson:gson:2.10.1'
 
     testImplementation(testFixtures(project(":nextflow")))
-    testImplementation "org.apache.groovy:groovy:4.0.25"
-    testImplementation "org.apache.groovy:groovy-nio:4.0.25"
+    testImplementation "org.apache.groovy:groovy:4.0.26"
+    testImplementation "org.apache.groovy:groovy-nio:4.0.26"
 }
 
 test {

diff --git a/plugins/nf-google/src/main/nextflow/cloud/google/batch/GoogleBatchTaskHandler.groovy b/plugins/nf-google/src/main/nextflow/cloud/google/batch/GoogleBatchTaskHandler.groovy
@@ -268,7 +268,7 @@ class GoogleBatchTaskHandler extends TaskHandler implements FusionAwareTask {
                     LifecyclePolicy.newBuilder()
                         .setActionCondition(
                             LifecyclePolicy.ActionCondition.newBuilder()
-                                .addExitCodes(50001)
+                                .addAllExitCodes(executor.config.autoRetryExitCodes)
                         )
                         .setAction(LifecyclePolicy.Action.RETRY_TASK)
                 )

diff --git a/plugins/nf-google/src/test/nextflow/cloud/google/batch/GoogleBatchTaskHandlerTest.groovy b/plugins/nf-google/src/test/nextflow/cloud/google/batch/GoogleBatchTaskHandlerTest.groovy
@@ -146,6 +146,7 @@ class GoogleBatchTaskHandlerTest extends Specification {
                 getBootDiskImage() >> BOOT_IMAGE
                 getCpuPlatform() >> CPU_PLATFORM
                 getMaxSpotAttempts() >> 5
+                getAutoRetryExitCodes() >> [50001,50002]
                 getSpot() >> true
                 getNetwork() >> 'net-1'
                 getServiceAccountEmail() >> 'foo@bar.baz'
@@ -198,7 +199,9 @@ class GoogleBatchTaskHandlerTest extends Specification {
         taskSpec.getMaxRunDuration().getSeconds() == TIMEOUT.seconds
         taskSpec.getVolumes(0).getMountPath() == '/tmp'
         taskSpec.getMaxRetryCount() == 5
+        taskSpec.getLifecyclePolicies(0).getActionCondition().getExitCodesCount() == 2
         taskSpec.getLifecyclePolicies(0).getActionCondition().getExitCodes(0) == 50001
+        taskSpec.getLifecyclePolicies(0).getActionCondition().getExitCodes(1) == 50002
         taskSpec.getLifecyclePolicies(0).getAction().toString() == 'RETRY_TASK'
         and:
         runnable.getContainer().getCommandsList().join(' ') == '/bin/bash -o pipefail -c bash .command.run'

diff --git a/plugins/nf-tower/build.gradle b/plugins/nf-tower/build.gradle
@@ -37,8 +37,8 @@ dependencies {
     api "com.fasterxml.jackson.core:jackson-databind:2.12.7.1"
 
     testImplementation(testFixtures(project(":nextflow")))
-    testImplementation "org.apache.groovy:groovy:4.0.25"
-    testImplementation "org.apache.groovy:groovy-nio:4.0.25"
+    testImplementation "org.apache.groovy:groovy:4.0.26"
+    testImplementation "org.apache.groovy:groovy-nio:4.0.26"
     // wiremock required by TowerFusionEnvTest
     testImplementation "org.wiremock:wiremock:3.5.4"
 }
diff --git a/plugins/nf-wave/build.gradle b/plugins/nf-wave/build.gradle
@@ -41,6 +41,6 @@ dependencies {
     api 'io.seqera:wave-utils:0.15.0'
 
     testImplementation(testFixtures(project(":nextflow")))
-    testImplementation "org.apache.groovy:groovy:4.0.25"
-    testImplementation "org.apache.groovy:groovy-nio:4.0.25"
+    testImplementation "org.apache.groovy:groovy:4.0.26"
+    testImplementation "org.apache.groovy:groovy-nio:4.0.26"
 }
diff --git a/tests/checks/fusion-symlink.nf/.checks b/tests/checks/fusion-symlink.nf/.checks
@@ -1,5 +1,43 @@
 #!/bin/bash
 
+# Retry a command with exponential backoff. The function returns 0 if the command was successful
+# on its first attempt, 1 if the command failed after all attempts, and 2 if the command was successful
+# after one or more retries.
+function _retry {
+
+  if [[ $# -lt 4 ]]; then
+    echo "Usage: _retry <max_attempts> <initial_delay> <max_delay> <cmd>"
+    return 1
+  fi
+
+  local max_attempts="$1"; shift
+  local initial_delay="$1"; shift
+  local max_delay="$1"; shift
+  local cmd=( "$@" )
+  local attempt_num=1
+  local max_attempts=${max_attempts}
+  local max_delay=${max_delay}
+  local initial_delay=${initial_delay}
+  local exit_code=0
+
+  until "${cmd[@]}"; do
+    exit_code=2
+    if (( attempt_num == max_attempts )); then
+      echo "-- [$attempt_num/$max_attempts] attempt failed! No more attempts left."
+      return 1
+    fi
+    echo "-- [$attempt_num/$max_attempts] attempt failed! Retrying in ${initial_delay}s..."
+    sleep "$initial_delay"
+    (( attempt_num++ ))
+    (( initial_delay *= 2 ))
+    if (( initial_delay > max_delay )); then
+      initial_delay=$max_delay
+    fi
+  done
+  echo "-- [$attempt_num/$max_attempts] attempt succeeded!"
+  return $exit_code
+}
+
 # Skip test if AWS keys are missing
 if [ -z "$AWS_ACCESS_KEY_ID" ]; then 
   echo "Missing AWS credentials -- Skipping test"
@@ -12,7 +50,11 @@ fi
 echo initial run
 $NXF_RUN -c .config
 
-$NXF_CMD fs cp s3://nextflow-ci/work/ci-test/fusion-symlink/data.txt data.txt
+_retry 5 1 16 "$NXF_CMD" fs cp s3://nextflow-ci/work/ci-test/fusion-symlink/data.txt data.txt
+if [ $? -eq 2 ]; then
+  echo "succeeded on retry"
+  false
+fi
 cmp data.txt .expected || false
 
 #
@@ -21,5 +63,10 @@ cmp data.txt .expected || false
 echo resumed run
 $NXF_RUN -c .config -resume
 
-$NXF_CMD fs cp s3://nextflow-ci/work/ci-test/fusion-symlink/data.txt data.txt
+_retry 5 1 16 "$NXF_CMD" fs cp s3://nextflow-ci/work/ci-test/fusion-symlink/data.txt data.txt
+if [ $? -eq 2 ]; then
+  echo "succeeded on retry"
+  false
+
+fi
 cmp data.txt .expected || false
diff --git a/tests/checks/fusion-symlink.nf/.config b/tests/checks/fusion-symlink.nf/.config
@@ -2,3 +2,4 @@ workDir = 's3://nextflow-ci/work'
 fusion.enabled = true
 fusion.exportStorageCredentials = true
 wave.enabled = true
+docker.runOptions = '-e FUSION_TRACING_DESTINATION=objectstore://'