Add eval output type (nextflow-io#4493)

This PR introduces the output `eval` type that allows the definition of
a script expression that needs to be computed in the script context to evaluate
the output value to be emitted. An example would be:

```
process someTask {
  output:
  eval 'bash --version'
  '''
  some-command --here
  '''
}
```

Signed-off-by: Paolo Di Tommaso <paolo.ditommaso@gmail.com>
Signed-off-by: Ben Sherman <bentshermann@gmail.com>
Co-authored-by: Ben Sherman <bentshermann@gmail.com>
Co-authored-by: Dr Marco Claudio De La Pierre <marco.delapierre@gmail.com>
Signed-off-by: Niklas Schandry <niklas@bio.lmu.de>

docs/process.md

@@ -1049,43 +1049,49 @@ To sum up, the use of output files with static names over dynamic ones is prefer
 
 The `env` qualifier allows you to output a variable defined in the process execution environment:
 
-```groovy
-process myTask {
-    output:
-    env FOO
-
-    script:
-    '''
-    FOO=$(ls -la)
-    '''
-}
-
-workflow {
-    myTask | view { "directory contents: $it" }
-}
+```{literalinclude} snippets/process-out-env.nf
+:language: groovy
 ```
 
+:::{versionchanged} 23.12.0-edge
+Prior to this version, if the environment variable contained multiple lines of output, the output would be compressed to a single line by converting newlines to spaces.
+:::
+
 (process-stdout)=
 
 ### Output type `stdout`
 
 The `stdout` qualifier allows you to output the `stdout` of the executed process:
 
-```groovy
-process sayHello {
-    output:
-    stdout
+```{literalinclude} snippets/process-stdout.nf
+:language: groovy
+```
 
-    """
-    echo Hello world!
-    """
-}
+(process-out-eval)=
 
-workflow {
-    sayHello | view { "I say... $it" }
-}
+### Output type `eval`
+
+:::{versionadded} 24.02.0-edge
+:::
+
+The `eval` qualifier allows you to capture the standard output of an arbitrary command evaluated the task shell interpreter context:
+
+```{literalinclude} snippets/process-out-eval.nf
+:language: groovy
 ```
 
+Only one-line Bash commands are supported. You can use a semi-colon `;` to specify multiple Bash commands on a single line, and many interpreters can execute arbitrary code on the command line, e.g. `python -c 'print("Hello world!")'`.
+
+If the command fails, the task will also fail. In Bash, you can append `|| true` to a command to suppress any command failure.
+
+(process-set)=
+
+### Output type `set`
+
+:::{deprecated} 19.08.1-edge
+Use `tuple` instead.
+:::
+
 (process-out-tuple)=
 
 ### Output type `tuple`

docs/snippets/process-out-env.nf

@@ -0,0 +1,13 @@
+process myTask {
+    output:
+    env FOO
+
+    script:
+    '''
+    FOO=$(ls -a)
+    '''
+}
+
+workflow {
+    myTask | view
+}

docs/snippets/process-out-env.out

@@ -0,0 +1,8 @@
+.
+..
+.command.begin
+.command.err
+.command.log
+.command.out
+.command.run
+.command.sh

docs/snippets/process-out-eval.nf

@@ -0,0 +1,12 @@
+process sayHello {
+    output:
+    eval('bash --version')
+
+    """
+    echo Hello world!
+    """
+}
+
+workflow {
+    sayHello | view
+}

docs/snippets/process-out-eval.out

@@ -0,0 +1,6 @@
+GNU bash, version 5.1.16(1)-release (x86_64-pc-linux-gnu)
+Copyright (C) 2020 Free Software Foundation, Inc.
+License GPLv3+: GNU GPL version 3 or later <http://gnu.org/licenses/gpl.html>
+
+This is free software; you are free to change and redistribute it.
+There is NO WARRANTY, to the extent permitted by law.

docs/snippets/process-stdout.nf

@@ -0,0 +1,12 @@
+process sayHello {
+    output:
+    stdout
+
+    """
+    echo Hello world!
+    """
+}
+
+workflow {
+    sayHello | view { "I say... $it" }
+}

docs/snippets/process-stdout.out

@@ -0,0 +1,2 @@
+I say... Hello world!
+

modules/nextflow/src/main/groovy/nextflow/ast/NextflowDSLImpl.groovy

@@ -27,6 +27,7 @@ import nextflow.script.BaseScript
 import nextflow.script.BodyDef
 import nextflow.script.IncludeDef
 import nextflow.script.TaskClosure
+import nextflow.script.TokenEvalCall
 import nextflow.script.TokenEnvCall
 import nextflow.script.TokenFileCall
 import nextflow.script.TokenPathCall
@@ -955,7 +956,7 @@ class NextflowDSLImpl implements ASTTransformation {
             def nested = methodCall.objectExpression instanceof MethodCallExpression
             log.trace "convert > output method: $methodName"
 
-            if( methodName in ['val','env','file','set','stdout','path','tuple'] && !nested ) {
+            if( methodName in ['val','env','eval','file','set','stdout','path','tuple'] && !nested ) {
                 // prefix the method name with the string '_out_'
                 methodCall.setMethod( new ConstantExpression('_out_' + methodName) )
                 fixMethodCall(methodCall)
@@ -1123,6 +1124,11 @@ class NextflowDSLImpl implements ASTTransformation {
                     return createX( TokenEnvCall, args )
                 }
 
+                if( methodCall.methodAsString == 'eval' && withinTupleMethod ) {
+                    def args = (TupleExpression) varToStrX(methodCall.arguments)
+                    return createX( TokenEvalCall, args )
+                }
+
                 /*
                  * input:
                  *   tuple val(x), .. from q

modules/nextflow/src/main/groovy/nextflow/exception/ProcessEvalException.groovy

@@ -0,0 +1,40 @@
+/*
+ * Copyright 2013-2023, Seqera Labs
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package nextflow.exception
+
+import groovy.transform.CompileStatic
+
+/**
+ * Exception thrown when a command output returns a non-zero exit status
+ * 
+ * @author Paolo Di Tommaso <paolo.ditommaso@gmail.com>
+ */
+@CompileStatic
+class ProcessEvalException extends RuntimeException implements ShowOnlyExceptionMessage {
+
+    String command
+    String output
+    int status
+
+    ProcessEvalException(String message, String command, String output, int status) {
+        super(message)
+        this.command = command
+        this.output = output
+        this.status = status
+    }
+}

modules/nextflow/src/main/groovy/nextflow/executor/BashWrapperBuilder.groovy

@@ -16,6 +16,8 @@
 
 package nextflow.executor
 
+import static java.nio.file.StandardOpenOption.*
+
 import java.nio.file.FileSystemException
 import java.nio.file.FileSystems
 import java.nio.file.Files
@@ -35,11 +37,7 @@ import nextflow.processor.TaskProcessor
 import nextflow.processor.TaskRun
 import nextflow.secret.SecretsLoader
 import nextflow.util.Escape
-
-import static java.nio.file.StandardOpenOption.*
-
 import nextflow.util.MemoryUnit
-
 /**
  * Builder to create the Bash script which is used to
  * wrap and launch the user task
@@ -177,20 +175,76 @@ class BashWrapperBuilder {
         }
     }
 
-    protected String getOutputEnvCaptureSnippet(List<String> names) {
-        def result = new StringBuilder()
-        result.append('\n')
-        result.append('# capture process environment\n')
-        result.append('set +u\n')
-        result.append('cd "$NXF_TASK_WORKDIR"\n')
-        for( int i=0; i<names.size(); i++) {
-            final key = names[i]
-            result.append "echo $key=\${$key[@]} "
-            result.append( i==0 ? '> ' : '>> ' )
-            result.append(TaskRun.CMD_ENV)
-            result.append('\n')
+    /**
+     * Generate a Bash script to be appended to the task command script
+     * that takes care of capturing the process output environment variables
+     * and evaluation commands
+     *
+     * @param outEnvs
+     *      The list of environment variables names whose value need to be captured
+     * @param outEvals
+     *      The set of commands to be evaluated to determine the output value to be captured
+     * @return
+     *      The Bash script to capture the output environment and eval commands
+     */
+    protected String getOutputEnvCaptureSnippet(List<String> outEnvs, Map<String,String> outEvals) {
+        // load the env template
+        final template = BashWrapperBuilder.class
+            .getResourceAsStream('command-env.txt')
+            .newReader()
+        final binding = Map.of('env_file', TaskRun.CMD_ENV)
+        final result = new StringBuilder()
+        result.append( engine.render(template, binding) )
+        appendOutEnv(result, outEnvs)
+        appendOutEval(result, outEvals)
+        return result.toString()
+    }
+
+    /**
+     * Render a Bash script to capture the one or more env variables
+     *
+     * @param result A {@link StringBuilder} instance to which append the result Bash script
+     * @param outEnvs The environment variables to be captured
+     */
+    protected void appendOutEnv(StringBuilder result, List<String> outEnvs) {
+        if( outEnvs==null )
+            outEnvs = List.<String>of()
+        // out env
+        for( String key : outEnvs ) {
+            result << "#\n"
+            result << "echo $key=\"\${$key[@]}\" >> ${TaskRun.CMD_ENV}\n"
+            result << "echo /$key/ >> ${TaskRun.CMD_ENV}\n"
+        }
+    }
+
+    /**
+     * Render a Bash script to capture the result of one or more commands
+     * evaluated in the task script context
+     *
+     * @param result
+     *      A {@link StringBuilder} instance to which append the result Bash script
+     * @param outEvals
+     *      A {@link Map} of key-value pairs modeling the commands to be evaluated;
+     *      where the key represents the environment variable (name) holding the
+     *      resulting output, and the pair value represent the Bash command to be
+     *      evaluated.
+     */
+    protected void appendOutEval(StringBuilder result, Map<String,String> outEvals) {
+        if( outEvals==null )
+            outEvals = Map.<String,String>of()
+        // out eval
+        for( Map.Entry<String,String> eval : outEvals ) {
+            result << "#\n"
+            result <<"nxf_eval_cmd STDOUT STDERR ${eval.value}\n"
+            result << 'status=$?\n'
+            result << 'if [ $status -eq 0 ]; then\n'
+            result << "  echo $eval.key=\"\$STDOUT\" >> ${TaskRun.CMD_ENV}\n"
+            result << "  echo /$eval.key/=exit:0 >> ${TaskRun.CMD_ENV}\n"
+            result << 'else\n'
+            result << "  echo $eval.key=\"\$STDERR\" >> ${TaskRun.CMD_ENV}\n"
+            result << "  echo /$eval.key/=exit:\$status >> ${TaskRun.CMD_ENV}\n"
+            result << 'fi\n'
         }
-        result.toString()
     }
 
     protected String stageCommand(String stagingScript) {
@@ -239,9 +293,16 @@ class BashWrapperBuilder {
          */
         final interpreter = TaskProcessor.fetchInterpreter(script)
 
-        if( outputEnvNames ) {
-            if( !isBash(interpreter) ) throw new IllegalArgumentException("Process output of type env is only allowed with Bash process command -- Current interpreter: $interpreter")
-            script += getOutputEnvCaptureSnippet(outputEnvNames)
+        /*
+         * append to the command script a prolog to capture the declared
+         * output environment (variable) and evaluation commands
+         */
+        if( outputEnvNames || outputEvals ) {
+            if( !isBash(interpreter) && outputEnvNames )
+                throw new IllegalArgumentException("Process output of type 'env' is only allowed with Bash process scripts -- Current interpreter: $interpreter")
+            if( !isBash(interpreter) && outputEvals )
+                throw new IllegalArgumentException("Process output of type 'eval' is only allowed with Bash process scripts -- Current interpreter: $interpreter")
+            script += getOutputEnvCaptureSnippet(outputEnvNames, outputEvals)
         }
 
         final binding = new HashMap<String,String>(20)

modules/nextflow/src/main/groovy/nextflow/processor/TaskBean.groovy

@@ -73,6 +73,8 @@ class TaskBean implements Serializable, Cloneable {
 
     List<String> outputEnvNames
 
+    Map<String,String> outputEvals
+
     String beforeScript
 
     String afterScript
@@ -144,6 +146,7 @@ class TaskBean implements Serializable, Cloneable {
 
         // stats
         this.outputEnvNames = task.getOutputEnvNames()
+        this.outputEvals = task.getOutputEvals()
         this.statsEnabled = task.getProcessor().getSession().statsEnabled
 
         this.inputFiles = task.getInputFilesMap()