Improve support for pipeline chaining

docs/strict-syntax.md

@@ -534,7 +534,7 @@ To enable these checks, set **Nextflow > Error reporting mode** to **paranoid**
 
 ### Using params outside the entry workflow
 
-While params can be used anywhere in the pipeline code, they are only intended to be used in the entry workflow.
+While params can be used anywhere in the pipeline code, they are only intended to be used in the entry workflow and the `output` block.
 
 As a best practice, processes and workflows should receive params as explicit inputs:
 

docs/workflow.md

@@ -49,13 +49,7 @@ params {
 }
 ```
 
-The following types can be used for parameters:
-
-- {ref}`stdlib-types-boolean`
-- {ref}`stdlib-types-float`
-- {ref}`stdlib-types-integer`
-- {ref}`stdlib-types-path`
-- {ref}`stdlib-types-string`
+All {ref}`standard types <stdlib-types>` except for the dataflow types (`Channel` and `Value`) can be used for parameters.
 
 Parameters can be used in the entry workflow:
 
@@ -66,12 +60,18 @@ workflow {
 ```
 
 :::{note}
-As a best practice, parameters should only be used directly in the entry workflow and passed to workflows and processes as explicit inputs.
+As a best practice, parameters should only be referenced in the entry workflow or `output` block. Parameters can be passed to workflows and processes as explicit inputs.
 :::
 
 The default value can be overridden by the command line, params file, or config file. Parameters from multiple sources are resolved in the order described in {ref}`cli-params`. Parameters specified on the command line are converted to the appropriate type based on the corresponding type annotation.
 
-A parameter that doesn't specify a default value is a *required* param. If a required param is not given a value at runtime, the run will fail.
+A parameter that doesn't specify a default value is a *required* parameter. If a required parameter is not given a value at runtime, the run will fail.
+
+Parameters with a collection type (i.e., `List`, `Set`, or `Bag`) can be supplied a file path instead of a literal collection. The file must be CSV, JSON, or YAML. Nextflow will parse the file contents and assign the resuling collection to the parameter. An error is thrown if the file contents do not match the parameter type.
+
+:::{note}
+When supplying a CSV file to a collection parameter, the CSV file must contain a header row and must use a comma (`,`) as the column separator.
+:::
 
 (workflow-params-legacy)=
 

modules/nextflow/src/main/groovy/nextflow/cli/CmdKubeRun.groovy

@@ -88,7 +88,7 @@ class CmdKubeRun extends CmdRun {
     @Override
     void run() {
         final scriptArgs = (args?.size()>1 ? args[1..-1] : []) as List<String>
-        final pipeline = stdin ? '-' : ( args ? args[0] : null )
+        final pipeline = args ? args[0] : null
         if( !pipeline )
             throw new AbortOperationException("No project name was specified")
         if( hasAnsiLogFlag() )

modules/nextflow/src/main/groovy/nextflow/cli/CmdRun.groovy

@@ -156,9 +156,6 @@ class CmdRun extends CmdBase implements HubOptions {
     @Parameter(names=['-latest'], description = 'Pull latest changes before run')
     boolean latest
 
-    @Parameter(names='-stdin', hidden = true)
-    boolean stdin
-
     @Parameter(names = ['-ansi'], hidden = true, arity = 0)
     void setAnsi(boolean value) {
         launcher.options.ansiLog = value
@@ -289,18 +286,10 @@ class CmdRun extends CmdBase implements HubOptions {
     @Override
     String getName() { NAME }
 
-    String getParamsFile() {
-        return paramsFile ?: sysEnv.get('NXF_PARAMS_FILE')
-    }
-
-    boolean hasParams() {
-        return params || getParamsFile()
-    }
-
     @Override
     void run() {
         final scriptArgs = (args?.size()>1 ? args[1..-1] : []) as List<String>
-        final pipeline = stdin ? '-' : ( args ? args[0] : null )
+        final pipeline = args ? args[0] : null
         if( !pipeline )
             throw new AbortOperationException("No project name was specified")
 
@@ -598,20 +587,6 @@ class CmdRun extends CmdBase implements HubOptions {
     protected ScriptFile getScriptFile0(String pipelineName) {
         assert pipelineName
 
-        /*
-         * read from the stdin
-         */
-        if( pipelineName == '-' ) {
-            def file = tryReadFromStdin()
-            if( !file )
-                throw new AbortOperationException("Cannot access `stdin` stream")
-
-            if( revision )
-                throw new AbortOperationException("Revision option cannot be used when running a script from stdin")
-
-            return new ScriptFile(file)
-        }
-
         /*
          * look for a file with the specified pipeline name
          */
@@ -661,171 +636,9 @@ class CmdRun extends CmdBase implements HubOptions {
 
     }
 
-    static protected File tryReadFromStdin() {
-        if( !System.in.available() )
-            return null
-
-        getScriptFromStream(System.in)
-    }
-
-    static protected File getScriptFromStream( InputStream input, String name = 'nextflow' ) {
-        input != null
-        File result = File.createTempFile(name, null)
-        result.deleteOnExit()
-        input.withReader { Reader reader -> result << reader }
-        return result
-    }
-
     @Memoized  // <-- avoid parse multiple times the same file and params
     Map parsedParams(Map configVars) {
-
-        final result = [:]
-
-        // apply params file
-        final file = getParamsFile()
-        if( file ) {
-            def path = validateParamsFile(file)
-            def type = path.extension.toLowerCase() ?: null
-            if( type == 'json' )
-                readJsonFile(path, configVars, result)
-            else if( type == 'yml' || type == 'yaml' )
-                readYamlFile(path, configVars, result)
-        }
-
-        // apply CLI params
-        if( !params )
-            return result
-
-        for( Map.Entry<String,String> entry : params ) {
-            addParam( result, entry.key, entry.value )
-        }
-        return result
-    }
-
-
-    static final private Pattern DOT_ESCAPED = ~/\\\./
-    static final private Pattern DOT_NOT_ESCAPED = ~/(?<!\\)\./
-
-    static protected void addParam(Map params, String key, String value, List path=[], String fullKey=null) {
-        if( !fullKey )
-            fullKey = key
-        final m = DOT_NOT_ESCAPED.matcher(key)
-        if( m.find() ) {
-            final p = m.start()
-            final root = key.substring(0, p)
-            if( !root ) throw new AbortOperationException("Invalid parameter name: $fullKey")
-            path.add(root)
-            def nested = params.get(root)
-            if( nested == null ) {
-                nested = new LinkedHashMap<>()
-                params.put(root, nested)
-            }
-            else if( nested !instanceof Map ) {
-                log.warn "Command line parameter --${path.join('.')} is overwritten by --${fullKey}"
-                nested = new LinkedHashMap<>()
-                params.put(root, nested)
-            }
-            addParam((Map)nested, key.substring(p+1), value, path, fullKey)
-        }
-        else {
-            addParam0(params, key.replaceAll(DOT_ESCAPED,'.'), parseParamValue(value))
-        }
-    }
-
-    static protected void addParam0(Map params, String key, Object value) {
-        if( key.contains('-') )
-            key = kebabToCamelCase(key)
-        params.put(key, value)
-    }
-
-    static protected String kebabToCamelCase(String str) {
-        final result = new StringBuilder()
-        str.split('-').eachWithIndex { String entry, int i ->
-            result << (i>0 ? StringUtils.capitalize(entry) : entry )
-        }
-        return result.toString()
-    }
-
-    static protected parseParamValue(String str) {
-        if ( SysEnv.get('NXF_DISABLE_PARAMS_TYPE_DETECTION') || NF.isSyntaxParserV2() )
-            return str
-
-        if ( str == null ) return null
-
-        if ( str.toLowerCase() == 'true') return Boolean.TRUE
-        if ( str.toLowerCase() == 'false' ) return Boolean.FALSE
-
-        if ( str==~/-?\d+(\.\d+)?/ && str.isInteger() ) return str.toInteger()
-        if ( str==~/-?\d+(\.\d+)?/ && str.isLong() ) return str.toLong()
-        if ( str==~/-?\d+(\.\d+)?/ && str.isDouble() ) return str.toDouble()
-
-        return str
-    }
-
-    private Path validateParamsFile(String file) {
-
-        def result = FileHelper.asPath(file)
-        def ext = result.getExtension()
-        if( !VALID_PARAMS_FILE.contains(ext) )
-            throw new AbortOperationException("Not a valid params file extension: $file -- It must be one of the following: ${VALID_PARAMS_FILE.join(',')}")
-
-        return result
-    }
-
-    static private Pattern PARAMS_VAR = ~/(?m)\$\{(\p{javaJavaIdentifierStart}\p{javaJavaIdentifierPart}*)}/
-
-    protected String replaceVars0(String content, Map binding) {
-        content.replaceAll(PARAMS_VAR) { List<String> matcher ->
-            // - the regex matcher is represented as list
-            // - the first element is the matching string ie. `${something}`
-            // - the second element is the group content ie. `something`
-            // - make sure the regex contains at least a group otherwise the closure
-            // parameter is a string instead of a list of the call fail
-            final placeholder = matcher.get(0)
-            final key = matcher.get(1)
-
-            if( !binding.containsKey(key) ) {
-                final msg = "Missing params file variable: $placeholder"
-                if(NF.strictMode)
-                    throw new AbortOperationException(msg)
-                log.warn msg
-                return placeholder
-            }
-
-            return binding.get(key)
-        }
-    }
-
-    private void readJsonFile(Path file, Map configVars, Map result) {
-        try {
-            def text = configVars ? replaceVars0(file.text, configVars) : file.text
-            def json = (Map<String,Object>) new JsonSlurper().parseText(text)
-            json.forEach((name, value) -> {
-                addParam0(result, name, value)
-            })
-        }
-        catch (NoSuchFileException | FileNotFoundException e) {
-            throw new AbortOperationException("Specified params file does not exist: ${file.toUriString()}")
-        }
-        catch( Exception e ) {
-            throw new AbortOperationException("Cannot parse params file: ${file.toUriString()} - Cause: ${e.message}", e)
-        }
-    }
-
-    private void readYamlFile(Path file, Map configVars, Map result) {
-        try {
-            def text = configVars ? replaceVars0(file.text, configVars) : file.text
-            def yaml = (Map<String,Object>) new Yaml().load(text)
-            yaml.forEach((name, value) -> {
-                addParam0(result, name, value)
-            })
-        }
-        catch (NoSuchFileException | FileNotFoundException e) {
-            throw new AbortOperationException("Specified params file does not exist: ${file.toUriString()}")
-        }
-        catch( Exception e ) {
-            throw new AbortOperationException("Cannot parse params file: ${file.toUriString()}", e)
-        }
+        return new ParamsCollector(params, paramsFile).apply(configVars)
     }
 
 }