Skip to content

Print process dictionary in logs (#5940) #5944

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
wants to merge 10 commits into
base: master
Choose a base branch
from
21 changes: 20 additions & 1 deletion modules/nextflow/src/main/groovy/nextflow/Session.groovy
Original file line number Diff line number Diff line change
Expand Up @@ -55,6 +55,7 @@ import nextflow.processor.TaskProcessor
import nextflow.script.BaseScript
import nextflow.script.ProcessConfig
import nextflow.script.ProcessFactory
import nextflow.script.ProcessDef
import nextflow.script.ScriptBinding
import nextflow.script.ScriptFile
import nextflow.script.ScriptMeta
Expand Down Expand Up @@ -870,7 +871,25 @@ class Session implements ISession {
if( enabled ) {
final names = ScriptMeta.allProcessNames()
final ver = "dsl${NF.dsl1 ?'1' :'2'}"
log.debug "Workflow process names [$ver]: ${names.join(', ')}"
final processDefs = ScriptMeta.allProcessDefinitions()
log.debug "Workflow process definitions [$ver]: ${processDefs.entrySet().collect{"${it.key} ${it.value}"}.join(', ')}"
// Add unaliased process names to the resolved names map
// Skip if script is undefined (for TESTING)
if(script) {
def resolvedNames = ScriptMeta.allResolvedProcessNames()
final mainScriptMeta = ScriptMeta.get(script)
mainScriptMeta.getLocalProcessNames().each { name ->
// The processes defined in the main script cannot be included in
// other scripts, as it would create a circular inclusion.
// Hence, no need to check for the key existence in the map.
def key = Map.entry(mainScriptMeta.getScriptPath(), name)
def list = new HashSet()
list.add(name)
resolvedNames.put(key, list)
}
log.debug "Resolved process names: ${resolvedNames.entrySet().join(', ')}"
}

validateConfig(names)
}
else {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -599,7 +599,14 @@ class TaskProcessor {
return
}
session.addIgniter {
log.debug "Starting process > $name"
// Closure to get the (nested) list of param_type:param_name
def getParamNamesAndTypes
getParamNamesAndTypes = { paramList ->
paramList.collect{param ->
"${(param instanceof InParam)? param.getTypeName() : param.class.simpleName}:${(param !instanceof TupleInParam && param !instanceof TupleOutParam)? param.name : "[${getParamNamesAndTypes(param.getInner())}]"}"}.join(', ')
}
// Print process name with associated input and outputs
log.debug "Starting process > $name (${getParamNamesAndTypes(getConfig().getInputs())}) -> (${getParamNamesAndTypes(getConfig().getOutputs())})"
op.start()
}
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,8 @@

package nextflow.script

import java.nio.file.Paths

import groovy.transform.CompileStatic
import groovy.util.logging.Slf4j
import nextflow.Const
Expand Down Expand Up @@ -127,7 +129,9 @@ class ProcessDef extends BindableDef implements IterableDef, ChainableDef {

@Override
ProcessDef cloneWithName(String name) {
ScriptMeta.addResolvedName(name)
final path = ScriptMeta.get(owner)?.getScriptPath()
final safePath = path?: Paths.get(".") // Default path for path-less scripts (for TESTING)
ScriptMeta.addResolvedName(name, safePath, this.baseName, processName)
def result = clone()
result.@processName = name
result.@simpleName = stripScope(name)
Expand Down
57 changes: 53 additions & 4 deletions modules/nextflow/src/main/groovy/nextflow/script/ScriptMeta.groovy
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@ package nextflow.script
import java.lang.reflect.Method
import java.lang.reflect.Modifier
import java.nio.file.Path
import java.nio.file.Paths

import groovy.transform.CompileStatic
import groovy.transform.Memoized
Expand Down Expand Up @@ -50,7 +51,7 @@ class ScriptMeta {

static private Map<Path,BaseScript> scriptsByPath = new HashMap<>(10)

static private Set<String> resolvedProcessNames = new HashSet<>(20)
static private Map<Map.Entry<Path,String>,Set<String>> resolvedProcessNames = new HashMap<>(20)

@TestOnly
static void reset() {
Expand All @@ -73,7 +74,7 @@ class ScriptMeta {
for( ScriptMeta entry : REGISTRY.values() )
result.addAll( entry.getProcessNames() )
// add all resolved names
result.addAll(resolvedProcessNames)
result.addAll(resolvedProcessNames.values().flatten())
return result
}

Expand All @@ -86,8 +87,50 @@ class ScriptMeta {
return result
}

static void addResolvedName(String name) {
resolvedProcessNames.add(name)
/**
* Returns a map of all local process definitions per script path.
* @return Map of script path and a list of process names
* defined in that script
* ie. [scriptPath -> [processName1, processName2]]
*/
static Map<Path, List<String>> allProcessDefinitions() {
final result = new HashMap()
for( final entry : REGISTRY.values() ) {
result.put(entry.getScriptPath(), entry.getLocalProcessNames())
}
return result
}

/**
* Returns a map of all process names and their aliases within the scripts.
* @return
*/
static Map<Map.Entry<Path, String>, Set<String>> allResolvedProcessNames() {
// Return a deep copy of the attribute, to avoid altering the static attribute.
def result = new HashMap(resolvedProcessNames.entrySet().size())
resolvedProcessNames.entrySet().each { entry ->
def key = Map.entry(entry.key.key, entry.key.value)
def value = new HashSet<String>(entry.value)
result.put(key, value)
}

return result
}

static void addResolvedName(String name, Path path, String baseName, String processName) {
def resolvedNameList = resolvedProcessNames.get(Map.entry(path, baseName))
if(!resolvedNameList){
resolvedNameList = new HashSet<String>()
resolvedProcessNames.put(Map.entry(path, baseName), resolvedNameList)
}

// If an aliased name is re-aliased, keep only the latest name.
// The first name is given on inclusion, the second when the
// actual workflow path is resolved. Only the second name appears
// in execution reports.
resolvedNameList.remove(processName)

resolvedNameList.add(name)
}

static Map<String,Path> allScriptNames() {
Expand Down Expand Up @@ -327,6 +370,12 @@ class ScriptMeta {
imports.put(name, component.cloneWithName(name))
}
else {
// Keep track of the resolved process name
if(component instanceof ProcessDef) {
final path = ScriptMeta.get(component.getOwner())?.getScriptPath()
final safePath = path?: Paths.get(".") // Default path for path-less scripts (for TESTING)
ScriptMeta.addResolvedName(name, safePath, component.getBaseName(), component.getName())
}
imports.put(name, component)
}
}
Expand Down
Loading