diff --git a/build.gradle b/build.gradle index c2d2570bba..aa71cf3b41 100644 --- a/build.gradle +++ b/build.gradle @@ -237,7 +237,7 @@ task compile { def getRuntimeConfigs() { def names = subprojects - .findAll { prj -> prj.name in ['nextflow','nf-commons','nf-httpfs','nf-lang','nf-lineage'] } + .findAll { prj -> prj.name in ['nextflow','nf-cli-v1','nf-commons','nf-httpfs','nf-lang','nf-lineage'] } .collect { it.name } FileCollection result = null @@ -262,8 +262,10 @@ task exportClasspath { doLast { def home = System.getProperty('user.home') def all = getRuntimeConfigs() - def libs = all.collect { File file -> /*println file.canonicalPath.replace(home, '$HOME');*/ file.canonicalPath; } - ['nextflow','nf-commons','nf-httpfs','nf-lang','nf-lineage'].each {libs << file("modules/$it/build/libs/${it}-${version}.jar").canonicalPath } + def libs = all.collect { File file -> file.canonicalPath } + ['nextflow','nf-cli-v1','nf-commons','nf-httpfs','nf-lang','nf-lineage'].each { + libs << file("modules/$it/build/libs/${it}-${version}.jar").canonicalPath + } file('.launch.classpath').text = libs.unique().join(':') } } @@ -276,7 +278,7 @@ ext.nexusEmail = project.findProperty('nexusEmail') // `signing.keyId` property needs to be defined in the `gradle.properties` file ext.enableSignArchives = project.findProperty('signing.keyId') -ext.coreProjects = projects( ':nextflow', ':nf-commons', ':nf-httpfs', ':nf-lang', ':nf-lineage' ) +ext.coreProjects = projects( ':nextflow', ':nf-cli-v1', ':nf-commons', ':nf-httpfs', ':nf-lang', ':nf-lineage' ) configure(coreProjects) { group = 'io.nextflow' diff --git a/modules/nextflow/build.gradle b/modules/nextflow/build.gradle index 57dfa8aa33..08081ae0b7 100644 --- a/modules/nextflow/build.gradle +++ b/modules/nextflow/build.gradle @@ -1,8 +1,19 @@ -plugins { - id "com.gradleup.shadow" version "8.3.5" -} +/* + * Copyright 2013-2025, Seqera Labs + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ apply plugin: 'groovy' -apply plugin: 'application' sourceSets { main.java.srcDirs = [] @@ -13,14 +24,6 @@ sourceSets { test.resources.srcDirs = ['src/test/resources'] } -compileGroovy { - options.compilerArgs = ['-XDignore.symbol.file'] -} - -configurations { - lineageImplementation -} - dependencies { api(project(':nf-commons')) api(project(':nf-httpfs')) @@ -41,14 +44,13 @@ dependencies { api "commons-lang:commons-lang:2.6" api "commons-codec:commons-codec:1.15" api "commons-io:commons-io:2.15.1" - api "com.beust:jcommander:1.35" api("com.esotericsoftware.kryo:kryo:2.24.0") { exclude group: 'com.esotericsoftware.minlog', module: 'minlog' } - api('org.iq80.leveldb:leveldb:0.12') - api('org.eclipse.jgit:org.eclipse.jgit:7.1.0.202411261347-r') - api ('javax.activation:activation:1.1.1') - api ('javax.mail:mail:1.4.7') - api ('org.yaml:snakeyaml:2.2') - api ('org.jsoup:jsoup:1.15.4') + api 'org.iq80.leveldb:leveldb:0.12' + api 'org.eclipse.jgit:org.eclipse.jgit:7.1.0.202411261347-r' + api 'javax.activation:activation:1.1.1' + api 'javax.mail:mail:1.4.7' + api 'org.yaml:snakeyaml:2.2' + api 'org.jsoup:jsoup:1.15.4' api 'jline:jline:2.9' api 'org.pf4j:pf4j:3.12.0' api 'dev.failsafe:failsafe:3.1.0' @@ -63,35 +65,4 @@ dependencies { testFixturesApi ("org.spockframework:spock-core:2.3-groovy-4.0") { exclude group: 'org.apache.groovy' } testFixturesApi ('org.spockframework:spock-junit4:2.3-groovy-4.0') { exclude group: 'org.apache.groovy' } testFixturesApi 'com.google.jimfs:jimfs:1.2' - // note: declare as separate dependency to avoid a circular dependency - lineageImplementation (project(':nf-lineage')) -} - - -test { - minHeapSize = "512m" - maxHeapSize = "4096m" -} - -application { - mainClass = 'nextflow.cli.Launcher' -} - -run{ - args( (project.hasProperty("runCmd") ? project.findProperty("runCmd") : "set a cmd to run").split(' ') ) -} - -shadowJar { - // add 'lineage' because it cannot be added to this project - // explicitly otherwise it will result into a circular dependency - configurations = [project.configurations.runtimeClasspath, project.configurations.lineageImplementation] - archiveClassifier='one' - manifest { - attributes 'Main-Class': "$mainClassName" - } - mergeServiceFiles() - mergeGroovyExtensionModules() - transform(com.github.jengelman.gradle.plugins.shadow.transformers.AppendingTransformer) { - resource = 'META-INF/extensions.idx' - } } diff --git a/modules/nextflow/src/main/groovy/nextflow/config/ConfigBuilder.groovy b/modules/nextflow/src/main/groovy/nextflow/config/ConfigBuilder.groovy index 0341f73523..cebb50958d 100644 --- a/modules/nextflow/src/main/groovy/nextflow/config/ConfigBuilder.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/config/ConfigBuilder.groovy @@ -16,133 +16,75 @@ package nextflow.config -import static nextflow.util.ConfigHelper.* - import java.nio.file.Path -import java.nio.file.Paths +import groovy.transform.CompileStatic import groovy.transform.Memoized -import groovy.transform.PackageScope import groovy.util.logging.Slf4j -import nextflow.Const import nextflow.NF -import nextflow.cli.CliOptions -import nextflow.cli.CmdConfig -import nextflow.cli.CmdNode -import nextflow.cli.CmdRun import nextflow.exception.AbortOperationException import nextflow.exception.ConfigParseException import nextflow.secret.SecretsLoader -import nextflow.trace.GraphObserver -import nextflow.trace.ReportObserver -import nextflow.trace.TimelineObserver -import nextflow.trace.TraceFileObserver -import nextflow.util.HistoryFile -import nextflow.util.SecretHelper /** * Builds up the Nextflow configuration object * * @author Paolo Di Tommaso */ @Slf4j +@CompileStatic class ConfigBuilder { - static final public String DEFAULT_PROFILE = 'standard' - - CliOptions options - - CmdRun cmdRun - - CmdNode cmdNode - - Path baseDir - - Path homeDir - - Path currentDir + public static final String DEFAULT_PROFILE = 'standard' - boolean showAllProfiles + protected Path baseDir - String profile = DEFAULT_PROFILE + protected Path currentDir - boolean validateProfile + protected Map params - List userConfigFiles = [] + protected String profile = DEFAULT_PROFILE - List parsedConfigFiles = [] + protected boolean validateProfile - boolean showClosures + protected boolean ignoreIncludes - boolean stripSecrets + protected boolean showAllProfiles - boolean showMissingVariables + protected boolean showClosures - Map emptyVariables = new LinkedHashMap<>(10) - - Map env = new HashMap<>(System.getenv()) - - List warnings = new ArrayList<>(10); - - { - setHomeDir(Const.APP_HOME_DIR) - setCurrentDir(Paths.get('.')) - } - - ConfigBuilder setShowClosures(boolean value) { - this.showClosures = value - return this - } + protected boolean showMissingVariables - ConfigBuilder setStripSecrets(boolean value) { - this.stripSecrets = value - return this - } + protected boolean stripSecrets - ConfigBuilder showMissingVariables(boolean value) { - this.showMissingVariables = value - return this - } + protected List parsedConfigFiles = [] - ConfigBuilder setOptions( CliOptions options ) { - this.options = options - return this - } + protected Map emptyVariables = [:] - ConfigBuilder setCmdRun( CmdRun cmdRun ) { - this.cmdRun = cmdRun - setProfile(cmdRun.profile) - return this - } + protected List warnings = [] - ConfigBuilder setBaseDir( Path path ) { + ConfigBuilder setBaseDir(Path path) { this.baseDir = path.complete() return this } - ConfigBuilder setCurrentDir( Path path ) { + ConfigBuilder setCurrentDir(Path path) { this.currentDir = path.complete() return this } - ConfigBuilder setHomeDir( Path path ) { - this.homeDir = path.complete() - return this - } - - ConfigBuilder setCmdNode( CmdNode node ) { - this.cmdNode = node + ConfigBuilder setIgnoreIncludes(boolean value) { + this.ignoreIncludes = value return this } - ConfigBuilder setCmdConfig( CmdConfig cmdConfig ) { - showAllProfiles = cmdConfig.showAllProfiles - setProfile(cmdConfig.profile) + ConfigBuilder setParams(Map params) { + this.params = params return this } - ConfigBuilder setProfile( String value ) { - profile = value ?: DEFAULT_PROFILE - validateProfile = value as boolean + ConfigBuilder setProfile(String value) { + this.profile = value ?: DEFAULT_PROFILE + this.validateProfile = value as boolean return this } @@ -151,179 +93,30 @@ class ConfigBuilder { return this } - ConfigBuilder setUserConfigFiles( Path... files ) { - setUserConfigFiles(files as List) + ConfigBuilder setShowClosures(boolean value) { + this.showClosures = value return this } - ConfigBuilder setUserConfigFiles( List files ) { - if( files ) - userConfigFiles.addAll(files) + ConfigBuilder setShowMissingVariables(boolean value) { + this.showMissingVariables = value return this } - static private wrapValue( value ) { - if( !value ) - return '' - - value = value.toString().trim() - if( value == 'true' || value == 'false') - return value - - if( value.isNumber() ) - return value - - return "'$value'" - } - - /** - * Transform the specified list of string to a list of files, verifying their existence. - *

- * If a file in the list does not exist an exception of type {@code CliArgumentException} is thrown. - *

- * If the specified list is empty it tries to return of default configuration files located at: - *

  • $HOME/.nextflow/taskConfig - *
  • $PWD/nextflow.taskConfig - * - * @param files - * @return - */ - @PackageScope - List validateConfigFiles( List files ) { - - def result = [] - if ( files ) { - for( String fileName : files ) { - def thisFile = currentDir.resolve(fileName) - if(!thisFile.exists()) { - throw new AbortOperationException("The specified configuration file does not exist: $thisFile -- check the name or choose another file") - } - result << thisFile - } - return result - } - - /* - * config file in the nextflow home - */ - def home = homeDir.resolve('config') - if( home.exists() ) { - log.debug "Found config home: $home" - result << home - } - - /** - * Config file in the pipeline base dir - * This config file name should be predictable, therefore cannot be overridden - */ - def base = null - if( baseDir && baseDir != currentDir ) { - base = baseDir.resolve('nextflow.config') - if( base.exists() ) { - log.debug "Found config base: $base" - result << base - } - } - - /** - * Local or user provided file - * Default config file name can be overridden with `NXF_CONFIG_FILE` env variable - */ - def configFileName = env.get('NXF_CONFIG_FILE') ?: 'nextflow.config' - def local = currentDir.resolve(configFileName) - if( local.exists() && local != base ) { - log.debug "Found config local: $local" - result << local - } - - def customConfigs = [] - if( userConfigFiles ) customConfigs.addAll(userConfigFiles) - if( options?.userConfig ) customConfigs.addAll(options.userConfig) - if( cmdRun?.runConfig ) customConfigs.addAll(cmdRun.runConfig) - if( customConfigs ) { - for( def item : customConfigs ) { - def configFile = item instanceof Path ? item : currentDir.resolve(item.toString()) - if(!configFile.exists()) { - throw new AbortOperationException("The specified configuration file does not exist: $configFile -- check the name or choose another file") - } - - log.debug "User config file: $configFile" - result << configFile - } - } - - return result + ConfigBuilder setStripSecrets(boolean value) { + this.stripSecrets = value + return this } - /** - * Create the nextflow configuration {@link ConfigObject} given a one or more - * config files - * - * @param files A list of config files {@link Path} - * @return The resulting {@link ConfigObject} instance - */ - @PackageScope - ConfigObject buildGivenFiles(List files) { - - final Map vars = cmdRun?.env - final boolean exportSysEnv = cmdRun?.exportSysEnv - - def items = [] - if( files ) for( Path file : files ) { - log.debug "Parsing config file: ${file.complete()}" - if (!file.exists()) { - log.warn "The specified configuration file cannot be found: $file" - } - else { - items << file - } - } - - Map env = [:] - if( exportSysEnv ) { - log.debug "Adding current system environment to session environment" - env.putAll(System.getenv()) - } - if( vars ) { - log.debug "Adding the following variables to session environment: $vars" - env.putAll(vars) - } - - // set the cluster options for the node command - if( cmdNode?.clusterOptions ) { - def str = new StringBuilder() - cmdNode.clusterOptions.each { k, v -> - str << "cluster." << k << '=' << wrapValue(v) << '\n' - } - items << str - } - - // -- add the executor obj from the command line args - if( cmdRun?.clusterOptions ) { - def str = new StringBuilder() - cmdRun.clusterOptions.each { k, v -> - str << "cluster." << k << '=' << wrapValue(v) << '\n' - } - items << str - } - - if( cmdRun?.executorOptions ) { - def str = new StringBuilder() - cmdRun.executorOptions.each { k, v -> - str << "executor." << k << '=' << wrapValue(v) << '\n' - } - items << str - } - - buildConfig0( env, items ) + List getParsedConfigFiles() { + return parsedConfigFiles } - @PackageScope - ConfigObject buildGivenFiles(Path... files) { - buildGivenFiles(files as List) + List getWarnings() { + return warnings } - protected Map configVars() { + Map configVars() { // this is needed to make sure to reuse the same // instance of the config vars across different instances of the ConfigBuilder // and prevent multiple parsing of the same params file (which can even be remote resource) @@ -331,45 +124,59 @@ class ConfigBuilder { } @Memoized - static private Map cacheableConfigVars(Path base) { + private static Map cacheableConfigVars(Path base) { final binding = new HashMap(10) binding.put('baseDir', base) binding.put('projectDir', base) - binding.put('launchDir', Paths.get('.').toRealPath()) - binding.put('outputDir', Paths.get('results').complete()) + binding.put('launchDir', Path.of('.').toRealPath()) + binding.put('outputDir', Path.of('results').complete()) binding.put('secrets', SecretsLoader.secretContext()) return binding } - protected ConfigObject buildConfig0( Map env, List configEntries ) { + /** + * Build a config object from the given config entries and + * environment variables. Each entry can be either a file (Path) + * or a snippet (String). + * + * @param env + * @param configEntries + */ + ConfigObject build(Map env=[:], List configEntries) { assert env != null - final ignoreIncludes = options ? options.ignoreConfigIncludes : false final parser = ConfigParserFactory.create() .setRenderClosureAsString(showClosures) .setStripSecrets(stripSecrets) .setIgnoreIncludes(ignoreIncludes) - ConfigObject result = new ConfigObject() - if( cmdRun && (cmdRun.hasParams()) ) - parser.setParams(cmdRun.parsedParams(configVars())) + if( params ) + parser.setParams(params) + + final result = new ConfigObject() + result.put('env', new ConfigObject()) // add the user specified environment to the session env - env.sort().each { name, value -> result.env.put(name,value) } + final resultEnv = (Map) result.env + env.sort().forEach((name, value) -> { + resultEnv.put(name, value) + }) if( configEntries ) { + final binding = [:] // the configuration object binds always the current environment // so that in the configuration file may be referenced any variable // in the current environment - final binding = new HashMap(System.getenv()) - binding.putAll(env) + if( NF.getSyntaxParserVersion() == 'v1' ) { + binding.putAll(System.getenv()) + binding.putAll(env) + } binding.putAll(configVars()) parser.setBinding(binding) // merge of the provided configuration files - for( def entry : configEntries ) { - + for( final entry : configEntries ) { try { merge0(result, parser, entry) } @@ -377,32 +184,34 @@ class ConfigBuilder { throw e } catch( Exception e ) { - def message = (entry instanceof Path ? "Unable to parse config file: '$entry'" : "Unable to parse configuration ") - throw new ConfigParseException(message,e) + final message = entry instanceof Path + ? "Unable to parse config file: '$entry'".toString() + : "Unable to parse configuration " + throw new ConfigParseException(message, e) } } + // validate profiles if specified if( validateProfile ) { checkValidProfile(parser.getProfiles()) } - } // guarantee top scopes - for( String name : ['env','session','params','process','executor']) { - if( !result.isSet(name) ) result.put(name, new ConfigObject()) + for( final name : List.of('env','executor','params','process') ) { + if( !result.isSet(name) ) + result.put(name, new ConfigObject()) } return result } /** - * Merge the main config with a separate config file + * Merge a config entry into an accumulated config object. * * @param result The main {@link ConfigObject} * @param parser The {@ConfigParser} instance - * @param entry The next config snippet/file to be parsed - * @return + * @param entry The next config file or snippet to parse */ protected void merge0(ConfigObject result, ConfigParser parser, entry) { if( !entry ) @@ -414,17 +223,21 @@ class ConfigBuilder { parser.setProfiles(profile.tokenize(',')) } - final config = parse0(parser, entry) + final config = parse0(parser,entry) if( NF.getSyntaxParserVersion() == 'v1' ) - validate(config, entry) + checkUnresolvedConfig(config,entry) result.merge(config) } + /** + * Parse a config file or snippet. + * + * @param parser + * @param entry + */ protected ConfigObject parse0(ConfigParser parser, entry) { if( entry instanceof File ) { - final path = entry.toPath() - parsedConfigFiles << path - return parser.parse(path) + return parse0(parser, entry.toPath()) } if( entry instanceof Path ) { @@ -440,17 +253,18 @@ class ConfigBuilder { } /** - * Validate a config object verifying is does not contains unresolved attributes + * Verify that a config object does not contain any unresolved attributes. * * @param config The {@link ConfigObject} to verify * @param file The source config file/snippet - * @return */ - protected void validate(ConfigObject config, file, String parent=null, List stack = new ArrayList()) { + protected void checkUnresolvedConfig(ConfigObject config, file, String parent=null, List stack = []) { for( String key : new ArrayList<>(config.keySet()) ) { final value = config.get(key) if( value instanceof ConfigObject ) { - final fqKey = parent ? "${parent}.${key}": key as String + final fqKey = parent + ? "${parent}.${key}".toString() + : key as String if( value.isEmpty() ) { final msg = "Unknown config attribute `$fqKey` -- check config file: $file".toString() if( showMissingVariables ) { @@ -466,7 +280,7 @@ class ConfigBuilder { stack.push(config) try { if( !stack.contains(value)) { - validate(value, file, fqKey, stack) + checkUnresolvedConfig(value, file, fqKey, stack) } else { log.debug("Found a recursive config property: `$fqKey`") @@ -490,18 +304,23 @@ class ConfigBuilder { } } - protected void checkValidProfile(Collection validNames) { + /** + * Validate that each specified profile exists in the merged config. + * + * @param declaredProfiles + */ + protected void checkValidProfile(Collection declaredProfiles) { if( !profile || profile == DEFAULT_PROFILE ) { return } - log.debug "Available config profiles: $validNames" + log.debug "Available config profiles: $declaredProfiles" for( String name : profile.tokenize(',') ) { - if( name in validNames ) + if( name in declaredProfiles ) continue def message = "Unknown configuration profile: '${name}'" - def choices = validNames.closest(name) + def choices = declaredProfiles.closest(name) if( choices ) { message += "\n\nDid you mean one of these?\n" choices.each { message += " ${it}\n" } @@ -511,413 +330,4 @@ class ConfigBuilder { throw new AbortOperationException(message) } } - - private String normalizeResumeId( String uniqueId ) { - if( !uniqueId ) - return null - if( uniqueId == 'last' || uniqueId == 'true' ) { - if( HistoryFile.disabled() ) - throw new AbortOperationException("The resume session id should be specified via `-resume` option when history file tracking is disabled") - uniqueId = HistoryFile.DEFAULT.getLast()?.sessionId - - if( !uniqueId ) { - log.warn "It appears you have never run this project before -- Option `-resume` is ignored" - } - } - - return uniqueId - } - - @PackageScope - void configRunOptions(ConfigObject config, Map env, CmdRun cmdRun) { - - // -- set config options - if( cmdRun.cacheable != null ) - config.cacheable = cmdRun.cacheable - - // -- set the run name - if( cmdRun.runName ) - config.runName = cmdRun.runName - - if( cmdRun.stubRun ) - config.stubRun = cmdRun.stubRun - - // -- set the output directory - if( cmdRun.outputDir ) - config.outputDir = cmdRun.outputDir - - if( cmdRun.preview ) - config.preview = cmdRun.preview - - // -- sets the working directory - if( cmdRun.workDir ) - config.workDir = cmdRun.workDir - - else if( !config.workDir ) - config.workDir = env.get('NXF_WORK') ?: 'work' - - if( cmdRun.bucketDir ) - config.bucketDir = cmdRun.bucketDir - - // -- sets the library path - if( cmdRun.libPath ) - config.libDir = cmdRun.libPath - - else if ( !config.isSet('libDir') && env.get('NXF_LIB') ) - config.libDir = env.get('NXF_LIB') - - // -- override 'process' parameters defined on the cmd line - cmdRun.process.each { name, value -> - config.process[name] = parseValue(value) - } - - if( cmdRun.withoutConda && config.conda instanceof Map ) { - // disable conda execution - log.debug "Disabling execution with Conda as requested by command-line option `-without-conda`" - config.conda.enabled = false - } - - // -- apply the conda environment - if( cmdRun.withConda ) { - if( cmdRun.withConda != '-' ) - config.process.conda = cmdRun.withConda - config.conda.enabled = true - } - - if( cmdRun.withoutSpack && config.spack instanceof Map ) { - // disable spack execution - log.debug "Disabling execution with Spack as requested by command-line option `-without-spack`" - config.spack.enabled = false - } - - // -- apply the spack environment - if( cmdRun.withSpack ) { - if( cmdRun.withSpack != '-' ) - config.process.spack = cmdRun.withSpack - config.spack.enabled = true - } - - // -- sets the resume option - if( cmdRun.resume ) - config.resume = cmdRun.resume - - if( config.isSet('resume') ) - config.resume = normalizeResumeId(config.resume as String) - - // -- sets `dumpHashes` option - if( cmdRun.dumpHashes ) { - config.dumpHashes = cmdRun.dumpHashes != '-' ? cmdRun.dumpHashes : 'default' - } - - if( cmdRun.dumpChannels ) - config.dumpChannels = cmdRun.dumpChannels.tokenize(',') - - // -- other configuration parameters - if( cmdRun.poolSize ) { - config.poolSize = cmdRun.poolSize - } - if( cmdRun.queueSize ) { - config.executor.queueSize = cmdRun.queueSize - } - if( cmdRun.pollInterval ) { - config.executor.pollInterval = cmdRun.pollInterval - } - - // -- sets trace file options - if( cmdRun.withTrace ) { - if( !(config.trace instanceof Map) ) - config.trace = [:] - config.trace.enabled = true - if( cmdRun.withTrace != '-' ) - config.trace.file = cmdRun.withTrace - else if( !config.trace.file ) - config.trace.file = TraceFileObserver.DEF_FILE_NAME - } - - // -- sets report report options - if( cmdRun.withReport ) { - if( !(config.report instanceof Map) ) - config.report = [:] - config.report.enabled = true - if( cmdRun.withReport != '-' ) - config.report.file = cmdRun.withReport - else if( !config.report.file ) - config.report.file = ReportObserver.DEF_FILE_NAME - } - - // -- sets timeline report options - if( cmdRun.withTimeline ) { - if( !(config.timeline instanceof Map) ) - config.timeline = [:] - config.timeline.enabled = true - if( cmdRun.withTimeline != '-' ) - config.timeline.file = cmdRun.withTimeline - else if( !config.timeline.file ) - config.timeline.file = TimelineObserver.DEF_FILE_NAME - } - - // -- sets DAG report options - if( cmdRun.withDag ) { - if( !(config.dag instanceof Map) ) - config.dag = [:] - config.dag.enabled = true - if( cmdRun.withDag != '-' ) - config.dag.file = cmdRun.withDag - else if( !config.dag.file ) - config.dag.file = GraphObserver.DEF_FILE_NAME - } - - if( cmdRun.withNotification ) { - if( !(config.notification instanceof Map) ) - config.notification = [:] - if( cmdRun.withNotification in ['true','false']) { - config.notification.enabled = cmdRun.withNotification == 'true' - } - else { - config.notification.enabled = true - config.notification.to = cmdRun.withNotification - } - } - - // -- sets the messages options - if( cmdRun.withWebLog ) { - log.warn "The command line option '-with-weblog' is deprecated - consider enabling this feature by setting 'weblog.enabled=true' in your configuration file" - if( !(config.weblog instanceof Map) ) - config.weblog = [:] - config.weblog.enabled = true - if( cmdRun.withWebLog != '-' ) - config.weblog.url = cmdRun.withWebLog - else if( !config.weblog.url ) - config.weblog.url = 'http://localhost' - } - - // -- sets tower options - if( cmdRun.withTower ) { - if( !(config.tower instanceof Map) ) - config.tower = [:] - config.tower.enabled = true - if( cmdRun.withTower != '-' ) - config.tower.endpoint = cmdRun.withTower - else if( !config.tower.endpoint ) - config.tower.endpoint = 'https://api.cloud.seqera.io' - } - - // -- set wave options - if( cmdRun.withWave ) { - if( !(config.wave instanceof Map) ) - config.wave = [:] - config.wave.enabled = true - if( cmdRun.withWave != '-' ) - config.wave.endpoint = cmdRun.withWave - else if( !config.wave.endpoint ) - config.wave.endpoint = 'https://wave.seqera.io' - } - - // -- set fusion options - if( cmdRun.withFusion ) { - if( !(config.fusion instanceof Map) ) - config.fusion = [:] - config.fusion.enabled = cmdRun.withFusion == 'true' - } - - // -- set cloudcache options - final envCloudPath = env.get('NXF_CLOUDCACHE_PATH') - if( cmdRun.cloudCachePath || envCloudPath ) { - if( !(config.cloudcache instanceof Map) ) - config.cloudcache = [:] - if( !config.cloudcache.isSet('enabled') ) - config.cloudcache.enabled = true - if( cmdRun.cloudCachePath && cmdRun.cloudCachePath != '-' ) - config.cloudcache.path = cmdRun.cloudCachePath - else if( !config.cloudcache.isSet('path') && envCloudPath ) - config.cloudcache.path = envCloudPath - } - - // -- add the command line parameters to the 'taskConfig' object - if( cmdRun.hasParams() ) - config.params = mergeMaps( (Map)config.params, cmdRun.parsedParams(configVars()), NF.strictMode ) - - if( cmdRun.withoutDocker && config.docker instanceof Map ) { - // disable docker execution - log.debug "Disabling execution in Docker container as requested by command-line option `-without-docker`" - config.docker.enabled = false - } - - if( cmdRun.withDocker ) { - configContainer(config, 'docker', cmdRun.withDocker) - } - - if( cmdRun.withPodman ) { - configContainer(config, 'podman', cmdRun.withPodman) - } - - if( cmdRun.withSingularity ) { - configContainer(config, 'singularity', cmdRun.withSingularity) - } - - if( cmdRun.withApptainer ) { - configContainer(config, 'apptainer', cmdRun.withApptainer) - } - - if( cmdRun.withCharliecloud ) { - configContainer(config, 'charliecloud', cmdRun.withCharliecloud) - } - } - - private void configContainer(ConfigObject config, String engine, def cli) { - log.debug "Enabling execution in ${engine.capitalize()} container as requested by command-line option `-with-$engine ${cmdRun.withDocker}`" - - if( !config.containsKey(engine) ) - config.put(engine, [:]) - - if( !(config.get(engine) instanceof Map) ) - throw new AbortOperationException("Invalid `$engine` definition in the config file") - - def containerConfig = (Map)config.get(engine) - containerConfig.enabled = true - if( cli != '-' ) { - // this is supposed to be a docker image name - config.process.container = cli - } - else if( containerConfig.image ) { - config.process.container = containerConfig.image - } - - if( !hasContainerDirective(config.process) ) - throw new AbortOperationException("You have requested to run with ${engine.capitalize()} but no image was specified") - - } - - /** - * Verify that configuration for process contains at last one `container` directive - * - * @param process - * @return {@code true} when a `container` is defined or {@code false} otherwise - */ - protected boolean hasContainerDirective(process) { - - if( process instanceof Map ) { - if( process.container ) - return true - - def result = process - .findAll { String name, value -> (name.startsWith('withName:') || name.startsWith('$')) && value instanceof Map } - .find { String name, Map value -> value.container as boolean } // the first non-empty `container` string - - return result as boolean - } - - return false - } - - ConfigObject buildConfigObject() { - // -- configuration file(s) - def configFiles = validateConfigFiles(options?.config) - def config = buildGivenFiles(configFiles) - - if( cmdRun ) - configRunOptions(config, System.getenv(), cmdRun) - - return config - } - - - /** - * @return A the application options hold in a {@code ConfigObject} instance - */ - ConfigMap build() { - toConfigMap(buildConfigObject()) - } - - protected static ConfigMap toConfigMap(ConfigObject config) { - assert config != null - (ConfigMap)normalize0((Map)config) - } - - static private normalize0( config ) { - - if( config instanceof Map ) { - ConfigMap result = new ConfigMap(config.size()) - for( String name : config.keySet() ) { - def value = (config as Map).get(name) - result.put(name, normalize0(value)) - } - return result - } - else if( config instanceof Collection ) { - List result = new ArrayList(config.size()) - for( entry in config ) { - result << normalize0(entry) - } - return result - } - else { - return config - } - } - - /** - * Merge two maps recursively avoiding keys to be overwritten - * - * @param config - * @param params - * @return a map resulting of merging result and right maps - */ - protected Map mergeMaps(Map config, Map params, boolean strict, List keys=[]) { - if( config==null ) - config = new LinkedHashMap() - - for( Map.Entry entry : params ) { - final key = entry.key.toString() - final value = entry.value - final previous = getConfigVal0(config, key) - keys << entry.key - - if( previous==null ) { - config[key] = value - } - else if( previous instanceof Map && value instanceof Map ) { - mergeMaps(previous, value, strict, keys) - } - else { - if( previous instanceof Map || value instanceof Map ) { - final msg = "Configuration setting type with key '${keys.join('.')}' does not match the parameter with the same key - Config value=$previous; parameter value=$value" - if(strict) - throw new AbortOperationException(msg) - log.warn(msg) - } - config[key] = value - } - } - - return config - } - - private Object getConfigVal0(Map config, String key) { - if( config instanceof ConfigObject ) { - return config.isSet(key) ? config.get(key) : null - } - else { - return config.get(key) - } - } - - static String resolveConfig(Path baseDir, CmdRun cmdRun) { - - final config = new ConfigBuilder() - .setShowClosures(true) - .setStripSecrets(true) - .setOptions(cmdRun.launcher.options) - .setCmdRun(cmdRun) - .setBaseDir(baseDir) - .buildConfigObject() - - // strip secret - SecretHelper.hideSecrets(config) - // compute config - final result = toCanonicalString(config, false) - // dump config for debugging - log.trace "Resolved config:\n${result.indent('\t')}" - return result - } } diff --git a/modules/nextflow/src/main/groovy/nextflow/scm/AssetManager.groovy b/modules/nextflow/src/main/groovy/nextflow/scm/AssetManager.groovy index b35f1a06d2..dc18c45857 100644 --- a/modules/nextflow/src/main/groovy/nextflow/scm/AssetManager.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/scm/AssetManager.groovy @@ -27,11 +27,11 @@ import groovy.transform.PackageScope import groovy.transform.ToString import groovy.transform.TupleConstructor import groovy.util.logging.Slf4j -import nextflow.cli.HubOptions -import nextflow.config.Manifest import nextflow.config.ConfigParserFactory +import nextflow.config.Manifest import nextflow.exception.AbortOperationException import nextflow.exception.AmbiguousPipelineNameException +import nextflow.scm.HubOptions import nextflow.script.ScriptFile import nextflow.util.IniFile import org.eclipse.jgit.api.CreateBranchCommand @@ -98,12 +98,12 @@ class AssetManager { * * @param pipeline The pipeline to be managed by this manager e.g. {@code nextflow-io/hello} */ - AssetManager( String pipelineName, HubOptions cliOpts = null) { + AssetManager( String pipelineName, HubOptions hubOpts = null) { assert pipelineName // read the default config file (if available) def config = ProviderConfig.getDefault() // build the object - build(pipelineName, config, cliOpts) + build(pipelineName, config, hubOpts) } AssetManager( String pipelineName, Map config ) { @@ -117,19 +117,19 @@ class AssetManager { * * @param pipelineName A project name or a project repository Git URL * @param config A {@link Map} holding the configuration properties defined in the {@link ProviderConfig#DEFAULT_SCM_FILE} file - * @param cliOpts User credentials provided on the command line. See {@link HubOptions} trait + * @param hubOpts User credentials provided on the command line. See {@link HubOptions} trait * @return The {@link AssetManager} object itself */ @PackageScope - AssetManager build( String pipelineName, Map config = null, HubOptions cliOpts = null ) { + AssetManager build( String pipelineName, Map config = null, HubOptions hubOpts = null ) { this.providerConfigs = ProviderConfig.createFromMap(config) this.project = resolveName(pipelineName) this.localPath = checkProjectDir(project) - this.hub = checkHubProvider(cliOpts) + this.hub = checkHubProvider(hubOpts) this.provider = createHubProvider(hub) - setupCredentials(cliOpts) + setupCredentials(hubOpts) validateProjectDir() return this @@ -152,14 +152,14 @@ class AssetManager { /** * Sets the user credentials on the {@link RepositoryProvider} object * - * @param cliOpts The user credentials specified on the program command line. See {@code HubOptions} + * @param hubOpts The user credentials specified on the program command line. See {@code HubOptions} */ @PackageScope - void setupCredentials( HubOptions cliOpts ) { - if( cliOpts?.hubUser ) { - cliOpts.hubProvider = hub - final user = cliOpts.getHubUser() - final pwd = cliOpts.getHubPassword() + void setupCredentials( HubOptions hubOpts ) { + if( hubOpts?.user() ) { + hubOpts = new HubOptions(hub, hubOpts.user()) + final user = hubOpts.getUser() + final pwd = hubOpts.getPassword() provider.setCredentials(user, pwd) } } @@ -214,15 +214,15 @@ class AssetManager { * Find out the "hub provider" (i.e. the platform on which the remote repository is stored * for example: github, bitbucket, etc) and verifies that it is a known provider. * - * @param cliOpts The user hub info provider as command line options. See {@link HubOptions} + * @param hubOpts The user hub info provider as command line options. See {@link HubOptions} * @return The name of hub name e.g. {@code github}, {@code bitbucket}, etc. */ @PackageScope - String checkHubProvider( HubOptions cliOpts ) { + String checkHubProvider( HubOptions hubOpts ) { def result = hub if( !result ) - result = cliOpts?.getHubProvider() + result = hubOpts?.provider() if( !result ) result = guessHubProviderFromGitConfig() if( !result ) diff --git a/modules/nextflow/src/main/groovy/nextflow/scm/HubOptions.groovy b/modules/nextflow/src/main/groovy/nextflow/scm/HubOptions.groovy new file mode 100644 index 0000000000..b199455aba --- /dev/null +++ b/modules/nextflow/src/main/groovy/nextflow/scm/HubOptions.groovy @@ -0,0 +1,54 @@ +/* + * Copyright 2013-2024, Seqera Labs + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package nextflow.scm + +import groovy.transform.CompileStatic +/** + * Options for interacting with a git repository provider i.e. GitHub or BitBucket + * + * @author Paolo Di Tommaso + */ +@CompileStatic +record HubOptions( + String provider, + String user +) { + + String getPassword() { + if( !user ) + return null + + final p = user.indexOf(':') + if( p != -1 ) + return user.substring(p + 1) + + final console = System.console() + if( !console ) + return null + + print "Enter your $provider password: " + return new String(console.readPassword()) + } + + String getUser() { + if( !user ) + return user + + final p = user.indexOf(':') + return p != -1 ? user.substring(0, p) : user + } +} diff --git a/modules/nextflow/src/main/groovy/nextflow/util/LoggerHelper.groovy b/modules/nextflow/src/main/groovy/nextflow/util/LoggerHelper.groovy index aaa1f18a27..3f02cac245 100644 --- a/modules/nextflow/src/main/groovy/nextflow/util/LoggerHelper.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/util/LoggerHelper.groovy @@ -56,8 +56,6 @@ import groovyx.gpars.dataflow.DataflowReadChannel import groovyx.gpars.dataflow.DataflowWriteChannel import nextflow.Global import nextflow.Session -import nextflow.cli.CliOptions -import nextflow.cli.Launcher import nextflow.exception.AbortOperationException import nextflow.exception.PlainExceptionMessage import nextflow.exception.ProcessException @@ -99,7 +97,7 @@ class LoggerHelper { static private LoggerHelper INSTANCE - private CliOptions opts + private LoggerOptions opts private boolean rolling = false @@ -148,7 +146,7 @@ class LoggerHelper { return this } - LoggerHelper(CliOptions opts) { + LoggerHelper(LoggerOptions opts) { this.opts = opts this.loggerContext = (LoggerContext) LoggerFactory.getILoggerFactory() } @@ -168,11 +166,11 @@ class LoggerHelper { } LoggerHelper setup() { - logFileName = opts.logFile ?: System.getenv('NXF_LOG_FILE') + logFileName = opts.logFile() ?: System.getenv('NXF_LOG_FILE') - final boolean quiet = opts.quiet - final List debugConf = opts.debug ?: new ArrayList() - final List traceConf = opts.trace ?: ( System.getenv('NXF_TRACE')?.tokenize(', ') ?: new ArrayList()) + final boolean quiet = opts.quiet() + final List debugConf = opts.debug() ?: new ArrayList() + final List traceConf = opts.trace() ?: ( System.getenv('NXF_TRACE')?.tokenize(', ') ?: new ArrayList()) // Reset all the logger final root = loggerContext.getLogger('ROOT') @@ -258,9 +256,9 @@ class LoggerHelper { protected Appender createConsoleAppender() { - final Appender result = daemon && opts.isBackground() + final Appender result = daemon && opts.background() ? (Appender) null - : (opts.ansiLog ? new CaptureAppender() : new ConsoleAppender()) + : (opts.ansiLog() ? new CaptureAppender() : new ConsoleAppender()) if( result ) { final filter = new ConsoleLoggerFilter( packages ) filter.setContext(loggerContext) @@ -357,11 +355,11 @@ class LoggerHelper { * @param traceConf The list of packages for which use a Trace logging level */ - static void configureLogger( Launcher launcher ) { - INSTANCE = new LoggerHelper(launcher.options) - .setDaemon(launcher.isDaemon()) + static void configureLogger(LoggerOptions opts, boolean daemon) { + INSTANCE = new LoggerHelper(opts) + .setDaemon(daemon) .setRolling(true) - .setSyslog(launcher.options.syslog) + .setSyslog(opts.syslog()) .setup() } diff --git a/modules/nextflow/src/main/groovy/nextflow/util/SpuriousDeps.groovy b/modules/nextflow/src/main/groovy/nextflow/util/LoggerOptions.groovy similarity index 56% rename from modules/nextflow/src/main/groovy/nextflow/util/SpuriousDeps.groovy rename to modules/nextflow/src/main/groovy/nextflow/util/LoggerOptions.groovy index dedbab6f86..7530d7f035 100644 --- a/modules/nextflow/src/main/groovy/nextflow/util/SpuriousDeps.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/util/LoggerOptions.groovy @@ -16,29 +16,16 @@ package nextflow.util - import groovy.transform.CompileStatic -import groovy.util.logging.Slf4j -import nextflow.cli.CmdBase -/** - * This class is used to resolve at runtime some spurious dependencies - * with optional modules - * - * @author Paolo Di Tommaso - */ -@Slf4j -@Deprecated -@CompileStatic -class SpuriousDeps { - - static CmdBase cmdCloud() { - try { - final clazz = Class.forName('nextflow.cli.CmdCloud') - return (CmdBase)clazz.newInstance() - } - catch (ClassNotFoundException e) { - return null - } - } +@CompileStatic +record LoggerOptions( + boolean ansiLog, + boolean background, + List debug, + String logFile, + boolean quiet, + String syslog, + List trace +) { } diff --git a/modules/nextflow/src/test/groovy/nextflow/config/ConfigBuilderTest.groovy b/modules/nextflow/src/test/groovy/nextflow/config/ConfigBuilderTest.groovy index ead6b64d59..b1df57246e 100644 --- a/modules/nextflow/src/test/groovy/nextflow/config/ConfigBuilderTest.groovy +++ b/modules/nextflow/src/test/groovy/nextflow/config/ConfigBuilderTest.groovy @@ -17,19 +17,13 @@ package nextflow.config import java.nio.file.Files -import java.nio.file.Paths +import java.nio.file.Path import nextflow.SysEnv -import nextflow.cli.CliOptions -import nextflow.cli.CmdConfig -import nextflow.cli.CmdNode -import nextflow.cli.CmdRun -import nextflow.cli.Launcher import nextflow.exception.AbortOperationException import nextflow.exception.ConfigParseException import nextflow.extension.FilesEx import nextflow.secret.SecretsLoader -import nextflow.trace.TraceHelper import nextflow.util.ConfigHelper import spock.lang.Ignore import spock.lang.Specification @@ -40,10 +34,6 @@ import spock.lang.Unroll */ class ConfigBuilderTest extends Specification { - def setup() { - TraceHelper.testTimestampFmt = '20221001' - } - def 'build config object' () { setup: @@ -51,7 +41,7 @@ class ConfigBuilderTest extends Specification { def builder = [:] as ConfigBuilder when: - def config = builder.buildConfig0(env,null) + def config = builder.build(env,null) then: ('PATH' in config.env ) @@ -80,8 +70,8 @@ class ConfigBuilderTest extends Specification { ''' when: - def config1 = builder.buildConfig0(env, [text1]) - def config2 = builder.buildConfig0(env, [text1, text2]) + def config1 = builder.build(env, [text1]) + def config2 = builder.build(env, [text1, text2]) // note: configuration object can be modified like any map config2.env ['ZZZ'] = '99' @@ -120,1386 +110,46 @@ class ConfigBuilderTest extends Specification { params.test = 2 ''' - when: - def config1 = builder.buildConfig0(env, [text1]) + def config1 = builder.build(env, [text1]) then: config1.task.field1 == 1 config1.task.field2 == 'hola' - config1.env.HOME == '/home/my:/some/path' - config1.env.PATH == '/local/bin' - config1.env.'dot.key.name' == 'any text' - - config1.params.test == 2 - config1.params.demo == 1 - - } - - def 'build config object 4' () { - - setup: - def builder = [:] as ConfigBuilder - builder.baseDir = Paths.get('/base/path') - - def text = ''' - params.p = "$baseDir/1" - params { - q = "$baseDir/2" - x = "$projectDir/3" - y = "$launchDir/4" - z = "$outputDir/5" - } - ''' - - when: - def cfg = builder.buildConfig0([:], [text]) - then: - cfg.params.p == '/base/path/1' - cfg.params.q == '/base/path/2' - cfg.params.x == '/base/path/3' - cfg.params.y == "${Paths.get('.').toRealPath()}/4" - cfg.params.z == "${Paths.get('results').complete()}/5" - - } - - def 'CLI params should override the ones defined in the config file' () { - setup: - def file = Files.createTempFile('test',null) - file.text = ''' - params { - alpha = 'x' - } - params.beta = 'y' - params.delta = 'Foo' - params.gamma = params.alpha - params { - omega = 'Bar' - } - - process { - publishDir = [path: params.alpha] - } - ''' - when: - def opt = new CliOptions() - def run = new CmdRun(params: [alpha: 'Hello', beta: 'World', omega: 'Last']) - def result = new ConfigBuilder().setOptions(opt).setCmdRun(run).buildGivenFiles(file) - - then: - result.params.alpha == 'Hello' // <-- params defined as CLI options override the ones in the config file - result.params.beta == 'World' // <-- as above - result.params.gamma == 'Hello' // <-- as above - result.params.omega == 'Last' - result.params.delta == 'Foo' - result.process.publishDir == [path: 'Hello'] - - cleanup: - file?.delete() - } - - def 'CLI params should override the ones defined in the config file [2]' () { - setup: - def file = Files.createTempFile('test',null) - file.text = ''' - params { - alpha = 'x' - beta = 'y' - delta = 'Foo' - gamma = params.alpha - omega = 'Bar' - } - - process { - publishDir = [path: params.alpha] - } - ''' - when: - def opt = new CliOptions() - def run = new CmdRun(params: [alpha: 'Hello', beta: 'World', omega: 'Last']) - def result = new ConfigBuilder().setOptions(opt).setCmdRun(run).buildGivenFiles(file) - - then: - result.params.alpha == 'Hello' // <-- params defined as CLI options override the ones in the config file - result.params.beta == 'World' // <-- as above - result.params.gamma == 'Hello' // <-- as above - result.params.omega == 'Last' - result.params.delta == 'Foo' - result.process.publishDir == [path: 'Hello'] - - cleanup: - file?.delete() - } - - - def 'CLI params should override the ones in one or more config files' () { - given: - def folder = File.createTempDir() - def configMain = new File(folder,'nextflow.config').absoluteFile - def snippet1 = new File(folder,'config1.txt').absoluteFile - def snippet2 = new File(folder,'config2.txt').absoluteFile - - - configMain.text = """ - process.name = 'alpha' - params.one = 'a' - params.xxx = 'x' - includeConfig "$snippet1" - """ - - snippet1.text = """ - params.two = 'b' - params.yyy = 'y' - - process.cpus = 4 - process.memory = '8GB' - - includeConfig("$snippet2") - """ - - snippet2.text = ''' - params.three = 'c' - params.zzz = 'z' - - process { disk = '1TB' } - process.resources.foo = 1 - process.resources.bar = 2 - ''' - - when: - def opt = new CliOptions() - def run = new CmdRun(params: [one: '1', two: 'dos', three: 'tres']) - def config = new ConfigBuilder().setOptions(opt).setCmdRun(run).buildGivenFiles(configMain.toPath()) - - then: - config.params.one == 1 - config.params.two == 'dos' - config.params.three == 'tres' - config.process.name == 'alpha' - config.params.xxx == 'x' - config.params.yyy == 'y' - config.params.zzz == 'z' - - config.process.cpus == 4 - config.process.memory == '8GB' - config.process.disk == '1TB' - config.process.resources.foo == 1 - config.process.resources.bar == 2 - - cleanup: - folder?.deleteDir() - } - - def 'should include config with params' () { - given: - def folder = File.createTempDir() - def configMain = new File(folder,'nextflow.config').absoluteFile - def snippet1 = new File(folder,'igenomes.config').absoluteFile - - - configMain.text = ''' - includeConfig 'igenomes.config' - ''' - - snippet1.text = ''' - params { - genomes { - 'GRCh37' { - fasta = "${params.igenomes_base}/genome.fa" - bwa = "${params.igenomes_base}/BWAIndex/genome.fa" - } - } - } - ''' - - when: - def opt = new CliOptions() - def run = new CmdRun(params: [igenomes_base: 'test']) - def config = new ConfigBuilder().setOptions(opt).setCmdRun(run).buildGivenFiles(configMain.toPath()) - - then: - config.params.genomes.GRCh37 == [fasta:'test/genome.fa', bwa:'test/BWAIndex/genome.fa'] - - cleanup: - folder?.deleteDir() - } - - - def 'should fetch the config path from env var' () { - given: - def folder = File.createTempDir() - def configMain = new File(folder,'my.config').absoluteFile - - - configMain.text = """ - process.name = 'alpha' - params.one = 'a' - params.two = 'b' - """ - - // relative path to current dir - when: - def config = new ConfigBuilder(env: [NXF_CONFIG_FILE: 'my.config']) .setCurrentDir(folder.toPath()) .build() - then: - config.params.one == 'a' - config.params.two == 'b' - config.process.name == 'alpha' - - // absolute path - when: - config = new ConfigBuilder(env: [NXF_CONFIG_FILE: configMain.toString()]) .build() - then: - config.params.one == 'a' - config.params.two == 'b' - config.process.name == 'alpha' - - // default should not find it - when: - config = new ConfigBuilder() .build() - then: - config.params == [:] - - cleanup: - folder?.deleteDir() - } - - def 'CLI params should overrides the ones in one or more profiles' () { - - setup: - def file = Files.createTempFile('test',null) - file.text = ''' - params.alpha = 'a' - params.beta = 'b' - params.delta = 'Foo' - params.gamma = params.alpha - - params { - genomes { - 'GRCh37' { - bed12 = '/data/genes.bed' - bismark = '/data/BismarkIndex' - bowtie = '/data/genome' - } - } - } - - profiles { - first { - params.alpha = 'Alpha' - params.omega = 'Omega' - params.gamma = 'First' - process.name = 'Bar' - } - - second { - params.alpha = 'xxx' - params.gamma = 'Second' - process { - publishDir = [path: params.alpha] - } - } - - } - - ''' - - - when: - def opt = new CliOptions() - def run = new CmdRun(params: [alpha: 'AAA', beta: 'BBB']) - def config = new ConfigBuilder().setOptions(opt).setCmdRun(run).buildGivenFiles(file) - then: - config.params.alpha == 'AAA' - config.params.beta == 'BBB' - config.params.delta == 'Foo' - config.params.gamma == 'AAA' - config.params.genomes.GRCh37.bed12 == '/data/genes.bed' - config.params.genomes.GRCh37.bismark == '/data/BismarkIndex' - config.params.genomes.GRCh37.bowtie == '/data/genome' - - when: - opt = new CliOptions() - run = new CmdRun(params: [alpha: 'AAA', beta: 'BBB'], profile: 'first') - config = new ConfigBuilder().setOptions(opt).setCmdRun(run).buildGivenFiles(file) - then: - config.params.alpha == 'AAA' - config.params.beta == 'BBB' - config.params.delta == 'Foo' - config.params.gamma == 'First' - config.process.name == 'Bar' - config.params.genomes.GRCh37.bed12 == '/data/genes.bed' - config.params.genomes.GRCh37.bismark == '/data/BismarkIndex' - config.params.genomes.GRCh37.bowtie == '/data/genome' - - - when: - opt = new CliOptions() - run = new CmdRun(params: [alpha: 'AAA', beta: 'BBB', genomes: 'xxx'], profile: 'second') - config = new ConfigBuilder().setOptions(opt).setCmdRun(run).buildGivenFiles(file) - then: - config.params.alpha == 'AAA' - config.params.beta == 'BBB' - config.params.delta == 'Foo' - config.params.gamma == 'Second' - config.params.genomes == 'xxx' - config.process.publishDir == [path: 'AAA'] - - cleanup: - file?.delete() - } - - def 'params-file should override params in the config file' () { - setup: - def baseDir = Paths.get('/my/base/dir') - and: - def params = Files.createTempFile('test', '.yml') - params.text = ''' - alpha: "Hello" - beta: "World" - omega: "Last" - theta: "${baseDir}/something" - '''.stripIndent() - and: - def file = Files.createTempFile('test',null) - file.text = ''' - params { - alpha = 'x' - } - params.beta = 'y' - params.delta = 'Foo' - params.gamma = params.alpha - params { - omega = 'Bar' - } - - process { - publishDir = [path: params.alpha] - } - ''' - when: - def opt = new CliOptions() - def run = new CmdRun(paramsFile: params) - def result = new ConfigBuilder().setOptions(opt).setCmdRun(run).setBaseDir(baseDir).buildGivenFiles(file) - - then: - result.params.alpha == 'Hello' // <-- params defined in the params-file overrides the ones in the config file - result.params.beta == 'World' // <-- as above - result.params.gamma == 'Hello' // <-- as above - result.params.omega == 'Last' - result.params.delta == 'Foo' - result.params.theta == "$baseDir/something" - result.process.publishDir == [path: 'Hello'] - - cleanup: - file?.delete() - params?.delete() - } - - def 'params should override params-file and override params in the config file' () { - setup: - def params = Files.createTempFile('test', '.yml') - params.text = ''' - alpha: "Hello" - beta: "World" - omega: "Last" - '''.stripIndent() - and: - def file = Files.createTempFile('test',null) - file.text = ''' - params { - alpha = 'x' - } - params.beta = 'y' - params.delta = 'Foo' - params.gamma = "I'm gamma" - params.omega = "I'm the last" - - process { - publishDir = [path: params.alpha] - } - ''' - when: - def opt = new CliOptions() - def run = new CmdRun(paramsFile: params, params: [alpha: 'Hola', beta: 'Mundo']) - def result = new ConfigBuilder().setOptions(opt).setCmdRun(run).buildGivenFiles(file) - - then: - result.params.alpha == 'Hola' // <-- this comes from the CLI - result.params.beta == 'Mundo' // <-- this comes from the CLI as well - result.params.omega == 'Last' // <-- this comes from the params-file - result.params.gamma == "I'm gamma" // <-- from the config - result.params.delta == 'Foo' // <-- from the config - result.process.publishDir == [path: 'Hola'] - - cleanup: - file?.delete() - params?.delete() - } - - def 'valid config files' () { - - given: - def folder = Files.createTempDirectory('test') - def f1 = folder.resolve('file1') - def f2 = folder.resolve('file2') - - when: - new ConfigBuilder() - .validateConfigFiles([f1.toString(), f2.toString()]) - then: - thrown(AbortOperationException) - - - when: - f1.text = '1'; f2.text = '2' - def files = new ConfigBuilder() - .validateConfigFiles([f1.toString(), f2.toString()]) - then: - files == [f1, f2] - - - when: - files = new ConfigBuilder(homeDir: folder, currentDir: folder) - .setUserConfigFiles(f1,f2) - .validateConfigFiles() - then: - files == [f1, f2] - - cleanup: - folder.deleteDir() - - } - - def 'should discover default config files' () { - given: - def homeDir = Files.createTempDirectory('home') - def baseDir = Files.createTempDirectory('work') - def workDir = Files.createTempDirectory('work') - - when: - def homeConfig = homeDir.resolve('config') - homeConfig.text = 'foo=1' - def files1 = new ConfigBuilder(homeDir: homeDir, baseDir: workDir, currentDir: workDir).validateConfigFiles() - then: - files1 == [homeConfig] - - when: - def workConfig = workDir.resolve('nextflow.config') - workConfig.text = 'bar=2' - def files2 = new ConfigBuilder(homeDir: homeDir, baseDir: workDir, currentDir: workDir).validateConfigFiles() - then: - files2 == [homeConfig, workConfig] - - when: - def baseConfig = baseDir.resolve('nextflow.config') - baseConfig.text = 'ciao=3' - def files3 = new ConfigBuilder(homeDir: homeDir, baseDir: baseDir, currentDir: workDir).validateConfigFiles() - then: - files3 == [homeConfig, baseConfig, workConfig] - - - cleanup: - homeDir?.deleteDir() - workDir?.deleteDir() - } - - def 'command executor options'() { - - when: - def opt = new CliOptions() - def run = new CmdRun(executorOptions: [ alpha: 1, 'beta.x': 'hola', 'beta.y': 'ciao' ]) - def result = new ConfigBuilder().setOptions(opt).setCmdRun(run).buildGivenFiles() - then: - result.executor.alpha == 1 - result.executor.beta.x == 'hola' - result.executor.beta.y == 'ciao' - - } - - def 'run command cluster options'() { - - when: - def opt = new CliOptions() - def run = new CmdRun(clusterOptions: [ alpha: 1, 'beta.x': 'hola', 'beta.y': 'ciao' ]) - def result = new ConfigBuilder().setOptions(opt).setCmdRun(run).buildGivenFiles() - then: - result.cluster.alpha == 1 - result.cluster.beta.x == 'hola' - result.cluster.beta.y == 'ciao' - - } - - def 'run with docker'() { - - when: - def opt = new CliOptions() - def run = new CmdRun(withDocker: 'cbcrg/piper') - def config = new ConfigBuilder().setOptions(opt).setCmdRun(run).build() - - then: - config.docker.enabled - config.process.container == 'cbcrg/piper' - - } - - def 'run with docker 2'() { - - given: - def file = Files.createTempFile('test','config') - file.deleteOnExit() - file.text = - ''' - docker { - image = 'busybox' - enabled = false - } - ''' - - when: - def opt = new CliOptions(config: [file.toFile().canonicalPath] ) - def run = new CmdRun(withDocker: '-') - def config = new ConfigBuilder().setOptions(opt).setCmdRun(run).build() - then: - config.docker.enabled - config.docker.image == 'busybox' - config.process.container == 'busybox' - - when: - opt = new CliOptions(config: [file.toFile().canonicalPath] ) - run = new CmdRun(withDocker: 'cbcrg/mybox') - config = new ConfigBuilder().setOptions(opt).setCmdRun(run).build() - then: - config.docker.enabled - config.process.container == 'cbcrg/mybox' - - } - - def 'run with docker 3'() { - given: - def file = Files.createTempFile('test','config') - file.deleteOnExit() - - when: - file.text = - ''' - process.'withName:test'.container = 'busybox' - ''' - def opt = new CliOptions(config: [file.toFile().canonicalPath]) - def run = new CmdRun(withDocker: '-') - def config = new ConfigBuilder().setOptions(opt).setCmdRun(run).build() - then: - config.docker.enabled - config.process.'withName:test'.container == 'busybox' - - when: - file.text = - ''' - process.container = 'busybox' - ''' - opt = new CliOptions(config: [file.toFile().canonicalPath]) - run = new CmdRun(withDocker: '-') - config = new ConfigBuilder().setOptions(opt).setCmdRun(run).build() - then: - config.docker.enabled - config.process.container == 'busybox' - - when: - opt = new CliOptions() - run = new CmdRun(withDocker: '-') - new ConfigBuilder().setOptions(opt).setCmdRun(run).build() - then: - def e = thrown(AbortOperationException) - e.message == 'You have requested to run with Docker but no image was specified' - - when: - file.text = - ''' - process.'withName:test'.tag = 'tag' - ''' - opt = new CliOptions(config: [file.toFile().canonicalPath]) - run = new CmdRun(withDocker: '-') - new ConfigBuilder().setOptions(opt).setCmdRun(run).build() - then: - e = thrown(AbortOperationException) - e.message == 'You have requested to run with Docker but no image was specified' - - } - - def 'run without docker'() { - - given: - def file = Files.createTempFile('test','config') - file.deleteOnExit() - file.text = - ''' - docker { - image = 'busybox' - enabled = true - } - ''' - - when: - def opt = new CliOptions(config: [file.toFile().canonicalPath] ) - def run = new CmdRun(withoutDocker: true) - def config = new ConfigBuilder().setOptions(opt).setCmdRun(run).build() - then: - !config.docker.enabled - config.docker.image == 'busybox' - !config.process.container - - } - - def 'config with cluster options'() { - - when: - def opt = new CliOptions() - def cmd = new CmdNode(clusterOptions: [join: 'x', group: 'y', interface: 'abc', slots: 10, 'tcp.alpha':'uno', 'tcp.beta': 'due']) - - def config = new ConfigBuilder() - .setOptions(opt) - .setCmdNode(cmd) - .build() - - then: - config.cluster.join == 'x' - config.cluster.group == 'y' - config.cluster.interface == 'abc' - config.cluster.slots == 10 - config.cluster.tcp.alpha == 'uno' - config.cluster.tcp.beta == 'due' - - } - - def 'has container directive' () { - when: - def config = new ConfigBuilder() - - then: - !config.hasContainerDirective(null) - !config.hasContainerDirective([:]) - !config.hasContainerDirective([foo: true]) - config.hasContainerDirective([container: 'hello/world']) - !config.hasContainerDirective([foo: 1, bar: 2]) - !config.hasContainerDirective([foo: 1, bar: 2, baz: [container: 'user/repo']]) - config.hasContainerDirective([foo: 1, bar: 2, $baz: [container: 'user/repo']]) - config.hasContainerDirective([foo: 1, bar: 2, 'withName:baz': [container: 'user/repo']]) - - } - - def 'should set session trace options' () { - - given: - def env = [:] - def builder = [:] as ConfigBuilder - - when: - def config = new ConfigObject() - builder.configRunOptions(config, env, new CmdRun()) - - then: - config.trace instanceof Map - !config.trace.enabled - !config.trace.file - - when: - config = new ConfigObject() - builder.configRunOptions(config, env, new CmdRun(withTrace: 'some-file')) - then: - config.trace instanceof Map - config.trace.enabled - config.trace.file == 'some-file' - - when: - config = new ConfigObject() - config.trace.file = 'foo.txt' - builder.configRunOptions(config, env, new CmdRun(withTrace: 'bar.txt')) - then: // command line should override the config file - config.trace instanceof Map - config.trace.enabled - config.trace.file == 'bar.txt' - - when: - config = new ConfigObject() - config.trace.file = 'foo.txt' - builder.configRunOptions(config, env, new CmdRun(withTrace: '-')) - then: // command line should override the config file - config.trace instanceof Map - config.trace.enabled - config.trace.file == 'foo.txt' - - when: - config = new ConfigObject() - builder.configRunOptions(config, env, new CmdRun(withTrace: '-')) - then: // command line should override the config file - config.trace instanceof Map - config.trace.enabled - config.trace.file == 'trace-20221001.txt' - } - - def 'should set session report options' () { - - given: - def env = [:] - def builder = [:] as ConfigBuilder - - when: - def config = new ConfigObject() - builder.configRunOptions(config, env, new CmdRun()) - then: - !config.report - - when: - config = new ConfigObject() - config.report.file = 'foo.html' - builder.configRunOptions(config, env, new CmdRun()) - then: - config.report instanceof Map - !config.report.enabled - config.report.file == 'foo.html' - - when: - config = new ConfigObject() - builder.configRunOptions(config, env, new CmdRun(withReport: 'my-report.html')) - then: - config.report instanceof Map - config.report.enabled - config.report.file == 'my-report.html' - - when: - config = new ConfigObject() - config.report.file = 'this-report.html' - builder.configRunOptions(config, env, new CmdRun(withReport: 'my-report.html')) - then: - config.report instanceof Map - config.report.enabled - config.report.file == 'my-report.html' - - when: - config = new ConfigObject() - config.report.file = 'this-report.html' - builder.configRunOptions(config, env, new CmdRun(withReport: '-')) - then: - config.report instanceof Map - config.report.enabled - config.report.file == 'this-report.html' - - when: - config = new ConfigObject() - builder.configRunOptions(config, env, new CmdRun(withReport: '-')) - then: - config.report instanceof Map - config.report.enabled - config.report.file == 'report-20221001.html' - } - - - def 'should set session dag options' () { - - given: - def env = [:] - def builder = [:] as ConfigBuilder - - when: - def config = new ConfigObject() - builder.configRunOptions(config, env, new CmdRun()) - then: - !config.dag - - when: - config = new ConfigObject() - config.dag.file = 'foo-dag.html' - builder.configRunOptions(config, env, new CmdRun()) - then: - config.dag instanceof Map - !config.dag.enabled - config.dag.file == 'foo-dag.html' - - when: - config = new ConfigObject() - builder.configRunOptions(config, env, new CmdRun(withDag: 'my-dag.html')) - then: - config.dag instanceof Map - config.dag.enabled - config.dag.file == 'my-dag.html' - - when: - config = new ConfigObject() - config.dag.file = 'this-dag.html' - builder.configRunOptions(config, env, new CmdRun(withDag: 'my-dag.html')) - then: - config.dag instanceof Map - config.dag.enabled - config.dag.file == 'my-dag.html' - - when: - config = new ConfigObject() - config.dag.file = 'this-dag.html' - builder.configRunOptions(config, env, new CmdRun(withDag: '-')) - then: - config.dag instanceof Map - config.dag.enabled - config.dag.file == 'this-dag.html' - - when: - config = new ConfigObject() - builder.configRunOptions(config, env, new CmdRun(withDag: '-')) - then: - config.dag instanceof Map - config.dag.enabled - config.dag.file == 'dag-20221001.html' - } - - def 'should set session weblog options' () { - - given: - def env = [:] - def builder = [:] as ConfigBuilder - - when: - def config = new ConfigObject() - builder.configRunOptions(config, env, new CmdRun()) - then: - !config.weblog - - when: - config = new ConfigObject() - config.weblog.url = 'http://bar.com' - builder.configRunOptions(config, env, new CmdRun()) - then: - config.weblog instanceof Map - !config.weblog.enabled - config.weblog.url == 'http://bar.com' - - when: - config = new ConfigObject() - builder.configRunOptions(config, env, new CmdRun(withWebLog: 'http://foo.com')) - then: - config.weblog instanceof Map - config.weblog.enabled - config.weblog.url == 'http://foo.com' - - when: - config = new ConfigObject() - config.weblog.enabled = true - config.weblog.url = 'http://bar.com' - builder.configRunOptions(config, env, new CmdRun(withWebLog: 'http://foo.com')) - then: - config.weblog instanceof Map - config.weblog.enabled - config.weblog.url == 'http://foo.com' - - when: - config = new ConfigObject() - config.weblog.enabled = true - config.weblog.url = 'http://bar.com' - builder.configRunOptions(config, env, new CmdRun(withWebLog: '-')) - then: - config.weblog instanceof Map - config.weblog.enabled - config.weblog.url == 'http://bar.com' - - when: - config = new ConfigObject() - config.weblog.enabled = true - builder.configRunOptions(config, env, new CmdRun(withWebLog: '-')) - then: - config.weblog instanceof Map - config.weblog.enabled - config.weblog.url == 'http://localhost' - - } - - def 'should set session timeline options' () { - - given: - def env = [:] - def builder = [:] as ConfigBuilder - - when: - def config = new ConfigObject() - builder.configRunOptions(config, env, new CmdRun()) - then: - !config.timeline - - when: - config = new ConfigObject() - config.timeline.file = 'my-file.html' - builder.configRunOptions(config, env, new CmdRun()) - then: - config.timeline instanceof Map - !config.timeline.enabled - config.timeline.file == 'my-file.html' - - when: - config = new ConfigObject() - builder.configRunOptions(config, env, new CmdRun(withTimeline: 'my-timeline.html')) - then: - config.timeline instanceof Map - config.timeline.enabled - config.timeline.file == 'my-timeline.html' - - when: - config = new ConfigObject() - config.timeline.enabled = true - config.timeline.file = 'this-timeline.html' - builder.configRunOptions(config, env, new CmdRun(withTimeline: 'my-timeline.html')) - then: - config.timeline instanceof Map - config.timeline.enabled - config.timeline.file == 'my-timeline.html' - - when: - config = new ConfigObject() - config.timeline.enabled = true - config.timeline.file = 'my-timeline.html' - builder.configRunOptions(config, env, new CmdRun(withTimeline: '-')) - then: - config.timeline instanceof Map - config.timeline.enabled - config.timeline.file == 'my-timeline.html' - - when: - config = new ConfigObject() - config.timeline.enabled = true - builder.configRunOptions(config, env, new CmdRun(withTimeline: '-')) - then: - config.timeline instanceof Map - config.timeline.enabled - config.timeline.file == 'timeline-20221001.html' - } - - def 'should set tower options' () { - - given: - def env = [:] - def builder = [:] as ConfigBuilder - - when: - def config = new ConfigObject() - builder.configRunOptions(config, env, new CmdRun()) - then: - !config.tower - - when: - config = new ConfigObject() - config.tower.endpoint = 'http://foo.com' - builder.configRunOptions(config, env, new CmdRun()) - then: - config.tower instanceof Map - !config.tower.enabled - config.tower.endpoint == 'http://foo.com' - - when: - config = new ConfigObject() - builder.configRunOptions(config, env, new CmdRun(withTower: 'http://bar.com')) - then: - config.tower instanceof Map - config.tower.enabled - config.tower.endpoint == 'http://bar.com' - - when: - config = new ConfigObject() - config.tower.endpoint = 'http://foo.com' - builder.configRunOptions(config, env, new CmdRun(withTower: '-')) - then: - config.tower instanceof Map - config.tower.enabled - config.tower.endpoint == 'http://foo.com' - - when: - config = new ConfigObject() - builder.configRunOptions(config, env, new CmdRun(withTower: '-')) - then: - config.tower instanceof Map - config.tower.enabled - config.tower.endpoint == 'https://api.cloud.seqera.io' - } - - def 'should set wave options' () { - - given: - def env = [:] - def builder = [:] as ConfigBuilder - - when: - def config = new ConfigObject() - builder.configRunOptions(config, env, new CmdRun()) - then: - !config.wave - - when: - config = new ConfigObject() - config.wave.endpoint = 'http://foo.com' - builder.configRunOptions(config, env, new CmdRun()) - then: - config.wave instanceof Map - !config.wave.enabled - config.wave.endpoint == 'http://foo.com' - - when: - config = new ConfigObject() - builder.configRunOptions(config, env, new CmdRun(withWave: 'http://bar.com')) - then: - config.wave instanceof Map - config.wave.enabled - config.wave.endpoint == 'http://bar.com' - - when: - config = new ConfigObject() - config.wave.endpoint = 'http://foo.com' - builder.configRunOptions(config, env, new CmdRun(withWave: '-')) - then: - config.wave instanceof Map - config.wave.enabled - config.wave.endpoint == 'http://foo.com' - - when: - config = new ConfigObject() - builder.configRunOptions(config, env, new CmdRun(withWave: '-')) - then: - config.wave instanceof Map - config.wave.enabled - config.wave.endpoint == 'https://wave.seqera.io' - } - - def 'should set cloudcache options' () { - - given: - def env = [:] - def builder = [:] as ConfigBuilder - - when: - def config = new ConfigObject() - builder.configRunOptions(config, env, new CmdRun()) - then: - !config.cloudcache - - when: - config = new ConfigObject() - config.cloudcache.path = 's3://foo/bar' - builder.configRunOptions(config, env, new CmdRun()) - then: - config.cloudcache instanceof Map - !config.cloudcache.enabled - config.cloudcache.path == 's3://foo/bar' - - when: - config = new ConfigObject() - builder.configRunOptions(config, env, new CmdRun(cloudCachePath: 's3://this/that')) - then: - config.cloudcache instanceof Map - config.cloudcache.enabled - config.cloudcache.path == 's3://this/that' - - when: - config = new ConfigObject() - builder.configRunOptions(config, env, new CmdRun(cloudCachePath: '-')) - then: - config.cloudcache instanceof Map - config.cloudcache.enabled - !config.cloudcache.path - - when: - config = new ConfigObject() - config.cloudcache.path = 's3://alpha/delta' - builder.configRunOptions(config, env, new CmdRun(cloudCachePath: '-')) - then: - config.cloudcache instanceof Map - config.cloudcache.enabled - config.cloudcache.path == 's3://alpha/delta' - - when: - config = new ConfigObject() - config.cloudcache.path = 's3://alpha/delta' - builder.configRunOptions(config, env, new CmdRun(cloudCachePath: 's3://should/override/config')) - then: - config.cloudcache instanceof Map - config.cloudcache.enabled - config.cloudcache.path == 's3://should/override/config' - - when: - config = new ConfigObject() - config.cloudcache.enabled = false - builder.configRunOptions(config, env, new CmdRun(cloudCachePath: 's3://should/override/config')) - then: - config.cloudcache instanceof Map - !config.cloudcache.enabled - config.cloudcache.path == 's3://should/override/config' - - when: - config = new ConfigObject() - builder.configRunOptions(config, [NXF_CLOUDCACHE_PATH:'s3://foo'], new CmdRun(cloudCachePath: 's3://should/override/env')) - then: - config.cloudcache instanceof Map - config.cloudcache.enabled - config.cloudcache.path == 's3://should/override/env' - - when: - config = new ConfigObject() - config.cloudcache.path = 's3://config/path' - builder.configRunOptions(config, [NXF_CLOUDCACHE_PATH:'s3://foo'], new CmdRun()) - then: - config.cloudcache instanceof Map - config.cloudcache.enabled - config.cloudcache.path == 's3://config/path' - - when: - config = new ConfigObject() - config.cloudcache.path = 's3://config/path' - builder.configRunOptions(config, [NXF_CLOUDCACHE_PATH:'s3://foo'], new CmdRun(cloudCachePath: 's3://should/override/config')) - then: - config.cloudcache instanceof Map - config.cloudcache.enabled - config.cloudcache.path == 's3://should/override/config' - - } - - def 'should enable conda env' () { - - given: - def env = [:] - def builder = [:] as ConfigBuilder - - when: - def config = new ConfigObject() - builder.configRunOptions(config, env, new CmdRun()) - then: - !config.conda - - when: - config = new ConfigObject() - config.conda.createOptions = 'something' - builder.configRunOptions(config, env, new CmdRun()) - then: - config.conda instanceof Map - !config.conda.enabled - config.conda.createOptions == 'something' - - when: - config = new ConfigObject() - builder.configRunOptions(config, env, new CmdRun(withConda: 'my-recipe.yml')) - then: - config.conda instanceof Map - config.conda.enabled - config.process.conda == 'my-recipe.yml' - - when: - config = new ConfigObject() - config.conda.enabled = true - builder.configRunOptions(config, env, new CmdRun(withConda: 'my-recipe.yml')) - then: - config.conda instanceof Map - config.conda.enabled - config.process.conda == 'my-recipe.yml' - - when: - config = new ConfigObject() - config.process.conda = 'my-recipe.yml' - builder.configRunOptions(config, env, new CmdRun(withConda: '-')) - then: - config.conda instanceof Map - config.conda.enabled - config.process.conda == 'my-recipe.yml' - } - - def 'should disable conda env' () { - given: - def file = Files.createTempFile('test','config') - file.deleteOnExit() - file.text = - ''' - conda { - enabled = true - } - ''' - - when: - def opt = new CliOptions(config: [file.toFile().canonicalPath] ) - def run = new CmdRun(withoutConda: true) - def config = new ConfigBuilder().setOptions(opt).setCmdRun(run).build() - then: - !config.conda.enabled - !config.process.conda - } - - def 'should enable spack env' () { - - given: - def env = [:] - def builder = [:] as ConfigBuilder - - when: - def config = new ConfigObject() - builder.configRunOptions(config, env, new CmdRun()) - then: - !config.spack - - when: - config = new ConfigObject() - config.spack.createOptions = 'something' - builder.configRunOptions(config, env, new CmdRun()) - then: - config.spack instanceof Map - !config.spack.enabled - config.spack.createOptions == 'something' - - when: - config = new ConfigObject() - builder.configRunOptions(config, env, new CmdRun(withSpack: 'my-recipe.yaml')) - then: - config.spack instanceof Map - config.spack.enabled - config.process.spack == 'my-recipe.yaml' - - when: - config = new ConfigObject() - config.spack.enabled = true - builder.configRunOptions(config, env, new CmdRun(withSpack: 'my-recipe.yaml')) - then: - config.spack instanceof Map - config.spack.enabled - config.process.spack == 'my-recipe.yaml' - - when: - config = new ConfigObject() - config.process.spack = 'my-recipe.yaml' - builder.configRunOptions(config, env, new CmdRun(withSpack: '-')) - then: - config.spack instanceof Map - config.spack.enabled - config.process.spack == 'my-recipe.yaml' - } - - def 'should disable spack env' () { - given: - def file = Files.createTempFile('test','config') - file.deleteOnExit() - file.text = - ''' - spack { - enabled = true - } - ''' - - when: - def opt = new CliOptions(config: [file.toFile().canonicalPath] ) - def run = new CmdRun(withoutSpack: true) - def config = new ConfigBuilder().setOptions(opt).setCmdRun(run).build() - then: - !config.spack.enabled - !config.process.spack - } - - def 'SHOULD SET `RESUME` OPTION'() { - - given: - def env = [:] - def builder = [:] as ConfigBuilder - - when: - def config = new ConfigObject() - builder.configRunOptions(config, env, new CmdRun()) - then: - !config.isSet('resume') - - when: - config = new ConfigObject() - config.resume ='alpha-beta-delta' - builder.configRunOptions(config, env, new CmdRun()) - then: - config.resume == 'alpha-beta-delta' - - when: - config = new ConfigObject() - config.resume ='alpha-beta-delta' - builder.configRunOptions(config, env, new CmdRun(resume: 'xxx-yyy')) - then: - config.resume == 'xxx-yyy' - - when: - config = new ConfigObject() - config.resume ='this-that' - builder.configRunOptions(config, env, new CmdRun(resume: 'xxx-yyy')) - then: - config.resume == 'xxx-yyy' - } - - def 'should set `workDir`' () { - - given: - def config = new ConfigObject() - def builder = [:] as ConfigBuilder - - when: - builder.configRunOptions(config, [:], new CmdRun()) - then: - config.workDir == 'work' - - when: - config = new ConfigObject() - builder.configRunOptions(config, [NXF_WORK: '/foo/bar'], new CmdRun()) - then: - config.workDir == '/foo/bar' - - when: - config = new ConfigObject() - config.workDir = 'hello/there' - builder.configRunOptions(config, [:], new CmdRun()) - then: - config.workDir == 'hello/there' - - when: - config = new ConfigObject() - config.workDir = 'hello/there' - builder.configRunOptions(config, [:], new CmdRun(workDir: 'my/work/dir')) - then: - config.workDir == 'my/work/dir' - } - - def 'should set `libDir`' () { - given: - def config = new ConfigObject() - def builder = [:] as ConfigBuilder - - when: - builder.configRunOptions(config, [:], new CmdRun()) - then: - !config.isSet('libDir') + config1.env.HOME == '/home/my:/some/path' + config1.env.PATH == '/local/bin' + config1.env.'dot.key.name' == 'any text' - when: - builder.configRunOptions(config, [NXF_LIB:'/foo/bar'], new CmdRun()) - then: - config.libDir == '/foo/bar' + config1.params.test == 2 + config1.params.demo == 1 - when: - builder.configRunOptions(config, [:], new CmdRun(libPath: 'my/lib/dir')) - then: - config.libDir == 'my/lib/dir' } - def 'should set `cacheable`' () { - given: - def env = [:] - def config + def 'build config object 4' () { + + setup: def builder = [:] as ConfigBuilder + builder.baseDir = Path.of('/base/path') - when: - config = new ConfigObject() - builder.configRunOptions(config, env, new CmdRun()) - then: - !config.isSet('cacheable') + def text = ''' + params.p = "$baseDir/1" + params { + q = "$baseDir/2" + x = "$projectDir/3" + y = "$launchDir/4" + z = "$outputDir/5" + } + ''' when: - config = new ConfigObject() - builder.configRunOptions(config, env, new CmdRun(cacheable: false)) + def cfg = builder.build([text]) then: - config.cacheable == false + cfg.params.p == '/base/path/1' + cfg.params.q == '/base/path/2' + cfg.params.x == '/base/path/3' + cfg.params.y == "${Path.of('.').toRealPath()}/4" + cfg.params.z == "${Path.of('results').complete()}/5" - when: - config = new ConfigObject() - builder.configRunOptions(config, env, new CmdRun(cacheable: true)) - then: - config.cacheable == true } def 'should check for a valid profile' () { @@ -1528,7 +178,6 @@ class ConfigBuilderTest extends Specification { """ .stripIndent().leftTrim() - when: builder.profile = 'delta' builder.checkValidProfile(['alpha','delta','omega']) @@ -1550,158 +199,6 @@ class ConfigBuilderTest extends Specification { } - def 'should set profile options' () { - - def builder - - when: - builder = new ConfigBuilder().setCmdRun(new CmdRun(profile: 'foo')) - then: - builder.profile == 'foo' - builder.validateProfile - - when: - builder = new ConfigBuilder().setCmdRun(new CmdRun()) - then: - builder.profile == 'standard' - !builder.validateProfile - - when: - builder = new ConfigBuilder().setCmdRun(new CmdRun(profile: 'standard')) - then: - builder.profile == 'standard' - builder.validateProfile - } - - def 'should set config options' () { - def builder - - when: - builder = new ConfigBuilder().setCmdConfig(new CmdConfig()) - then: - !builder.showAllProfiles - - when: - builder = new ConfigBuilder().setCmdConfig(new CmdConfig(showAllProfiles: true)) - then: - builder.showAllProfiles - - when: - builder = new ConfigBuilder().setCmdConfig(new CmdConfig(profile: 'foo')) - then: - builder.profile == 'foo' - builder.validateProfile - - } - - def 'should set params into config object' () { - - given: - def emptyFile = Files.createTempFile('empty','config').toFile() - def EMPTY = [emptyFile.toString()] - - def configFile = Files.createTempFile('test','config').toFile() - configFile.deleteOnExit() - configFile.text = ''' - params.foo = 1 - params.bar = 2 - params.data = '/some/path' - ''' - configFile = configFile.toString() - - def jsonFile = Files.createTempFile('test','.json').toFile() - jsonFile.text = ''' - { - "foo": 10, - "bar": 20 - } - ''' - jsonFile = jsonFile.toString() - - def yamlFile = Files.createTempFile('test','.yaml').toFile() - yamlFile.text = ''' - { - "foo": 100, - "bar": 200 - } - ''' - yamlFile = yamlFile.toString() - - def config - - when: - config = new ConfigBuilder().setOptions(new CliOptions(config: EMPTY)).setCmdRun(new CmdRun()).build() - then: - config.params == [:] - - // get params for the CLI - when: - config = new ConfigBuilder().setOptions(new CliOptions(config: EMPTY)).setCmdRun(new CmdRun(params: [foo:'one', bar:'two'])).build() - then: - config.params == [foo:'one', bar:'two'] - - // get params from config file - when: - config = new ConfigBuilder().setOptions(new CliOptions(config: [configFile])).setCmdRun(new CmdRun()).build() - then: - config.params == [foo:1, bar:2, data: '/some/path'] - - // get params form JSON file - when: - config = new ConfigBuilder().setOptions(new CliOptions(config: EMPTY)).setCmdRun(new CmdRun(paramsFile: jsonFile)).build() - then: - config.params == [foo:10, bar:20] - - // get params from YAML file - when: - config = new ConfigBuilder().setOptions(new CliOptions(config: EMPTY)).setCmdRun(new CmdRun(paramsFile: yamlFile)).build() - then: - config.params == [foo:100, bar:200] - - // cli override config - when: - config = new ConfigBuilder().setOptions(new CliOptions(config: [configFile])).setCmdRun(new CmdRun(params:[foo:'hello', baz:'world'])).build() - then: - config.params == [foo:'hello', bar:2, baz: 'world', data: '/some/path'] - - // CLI override JSON - when: - config = new ConfigBuilder().setOptions(new CliOptions(config: EMPTY)).setCmdRun(new CmdRun(params:[foo:'hello', baz:'world'], paramsFile: jsonFile)).build() - then: - config.params == [foo:'hello', bar:20, baz: 'world'] - - // JSON override config - when: - config = new ConfigBuilder().setOptions(new CliOptions(config: [configFile])).setCmdRun(new CmdRun(paramsFile: jsonFile)).build() - then: - config.params == [foo:10, bar:20, data: '/some/path'] - - - // CLI override JSON that override config - when: - config = new ConfigBuilder().setOptions(new CliOptions(config: [configFile])).setCmdRun(new CmdRun(paramsFile: jsonFile, params: [foo:'Ciao'])).build() - then: - config.params == [foo:'Ciao', bar:20, data: '/some/path'] - } - - def 'should run with conda' () { - - when: - def config = new ConfigBuilder().setCmdRun(new CmdRun(withConda: '/some/path/env.yml')).build() - then: - config.process.conda == '/some/path/env.yml' - - } - - def 'should run with spack' () { - - when: - def config = new ConfigBuilder().setCmdRun(new CmdRun(withSpack: '/some/path/env.yaml')).build() - then: - config.process.spack == '/some/path/env.yaml' - - } - def 'should warn about missing attribute' () { given: @@ -1712,10 +209,8 @@ class ConfigBuilderTest extends Specification { params.foo = HOME ''' - when: - def opt = new CliOptions(config: [file.toFile().canonicalPath] ) - def cfg = new ConfigBuilder().setOptions(opt).build() + def cfg = new ConfigBuilder().build([file]) then: cfg.params.foo == System.getenv('HOME') @@ -1724,8 +219,7 @@ class ConfigBuilderTest extends Specification { ''' params.foo = bar ''' - opt = new CliOptions(config: [file.toFile().canonicalPath] ) - new ConfigBuilder().setOptions(opt).build() + new ConfigBuilder().build([file]) then: def e = thrown(ConfigParseException) e.message == "Unknown config attribute `bar` -- check config file: ${file.toRealPath()}".toString() @@ -1743,11 +237,9 @@ class ConfigBuilderTest extends Specification { ''' when: - def opt = new CliOptions(config: [file.toFile().canonicalPath] ) def builder = new ConfigBuilder() - .setOptions(opt) - .showMissingVariables(true) - def cfg = builder.buildConfigObject() + .setShowMissingVariables(true) + def cfg = builder.build([file]) def str = ConfigHelper.toCanonicalString(cfg) then: str == '''\ @@ -1772,15 +264,13 @@ class ConfigBuilderTest extends Specification { ''' params.x = foo.bar ''' - def opt = new CliOptions(config: [file.toFile().canonicalPath] ) - new ConfigBuilder().setOptions(opt).build() + new ConfigBuilder().build([file]) then: def e = thrown(ConfigParseException) e.message == "Unknown config attribute `foo.bar` -- check config file: ${file.toRealPath()}".toString() } - def 'should collect config files' () { given: @@ -1806,91 +296,6 @@ class ConfigBuilderTest extends Specification { file2?.delete() } - - def 'should configure notification' () { - - given: - Map config - - when: - config = new ConfigBuilder().setCmdRun(new CmdRun()).build() - then: - !config.notification - - when: - config = new ConfigBuilder().setCmdRun(new CmdRun(withNotification: true)).build() - then: - config.notification.enabled == true - - when: - config = new ConfigBuilder().setCmdRun(new CmdRun(withNotification: false)).build() - then: - config.notification.enabled == false - config.notification.to == null - - when: - config = new ConfigBuilder().setCmdRun(new CmdRun(withNotification: 'yo@nextflow.com')).build() - then: - config.notification.enabled == true - config.notification.to == 'yo@nextflow.com' - } - - def 'should configure fusion' () { - - given: - Map config - - when: - config = new ConfigBuilder().setCmdRun(new CmdRun()).build() - then: - !config.fusion - - when: - config = new ConfigBuilder().setCmdRun(new CmdRun(withFusion: true)).build() - then: - config.fusion.enabled == true - - when: - config = new ConfigBuilder().setCmdRun(new CmdRun(withFusion: false)).build() - then: - config.fusion == [enabled: false] - - when: - config = new ConfigBuilder().setCmdRun(new CmdRun(withFusion: true)).build() - then: - config.fusion == [enabled: true] - } - - def 'should configure stub run mode' () { - given: - Map config - - when: - config = new ConfigBuilder().setCmdRun(new CmdRun()).build() - then: - !config.stubRun - - when: - config = new ConfigBuilder().setCmdRun(new CmdRun(stubRun: true)).build() - then: - config.stubRun == true - } - - def 'should configure preview mode' () { - given: - Map config - - when: - config = new ConfigBuilder().setCmdRun(new CmdRun()).build() - then: - !config.preview - - when: - config = new ConfigBuilder().setCmdRun(new CmdRun(preview: true)).build() - then: - config.preview == true - } - def 'should merge profiles' () { given: def ENV = [:] @@ -1927,7 +332,7 @@ class ConfigBuilderTest extends Specification { when: builder.profile = 'cfg1' - result = builder.buildConfig0(ENV, [CONFIG]) + result = builder.build(ENV, [CONFIG]) then: result.process.container == 'base' result.process.executor == 'sge' @@ -1935,7 +340,7 @@ class ConfigBuilderTest extends Specification { when: builder.profile = 'cfg2' - result = builder.buildConfig0(ENV, [CONFIG]) + result = builder.build(ENV, [CONFIG]) then: result.process.container == 'base' result.process.executor == 'batch' @@ -1943,7 +348,7 @@ class ConfigBuilderTest extends Specification { when: builder.profile = 'cfg1,docker' - result = builder.buildConfig0(ENV, [CONFIG]) + result = builder.build(ENV, [CONFIG]) then: result.process.container == 'foo/1' result.process.executor == 'sge' @@ -1953,7 +358,7 @@ class ConfigBuilderTest extends Specification { when: builder.profile = 'cfg1,singularity' - result = builder.buildConfig0(ENV, [CONFIG]) + result = builder.build(ENV, [CONFIG]) then: result.process.container == 'bar-2.img' result.process.executor == 'sge' @@ -1963,7 +368,7 @@ class ConfigBuilderTest extends Specification { when: builder.profile = 'cfg2,singularity' - result = builder.buildConfig0(ENV, [CONFIG]) + result = builder.build(ENV, [CONFIG]) then: result.process.container == 'bar-2.img' result.process.executor == 'batch' @@ -1973,7 +378,7 @@ class ConfigBuilderTest extends Specification { when: builder.profile = 'missing' - builder.buildConfig0(ENV, [CONFIG]) + builder.build(ENV, [CONFIG]) then: thrown(AbortOperationException) } @@ -2012,7 +417,7 @@ class ConfigBuilderTest extends Specification { when: builder.showAllProfiles = true - result = builder.buildConfig0(ENV, [CONFIG]) + result = builder.build(ENV, [CONFIG]) then: result.profiles.cfg1.process.executor == 'sge' result.profiles.cfg1.process.queue == 'short' @@ -2045,7 +450,7 @@ class ConfigBuilderTest extends Specification { ''' when: - def cfg1 = new ConfigBuilder().buildConfig0([:], [file1]) + def cfg1 = new ConfigBuilder().build([file1]) then: cfg1.process.cpus == 2 cfg1.process.'withName:bar'.cpus == 4 @@ -2059,7 +464,7 @@ class ConfigBuilderTest extends Specification { includeConfig "$file1" """ - def cfg2 = new ConfigBuilder().buildConfig0([:], [file2]) + def cfg2 = new ConfigBuilder().build([file2]) then: cfg1 == cfg2 @@ -2085,7 +490,7 @@ class ConfigBuilderTest extends Specification { ''' when: - def cfg1 = new ConfigBuilder().buildConfig0([:], [file1]) + def cfg1 = new ConfigBuilder().build([file1]) then: cfg1.process.cpus == 1 cfg1.process.ext.args == 'Hello World!' @@ -2116,7 +521,7 @@ class ConfigBuilderTest extends Specification { ''' when: - def cfg1 = new ConfigBuilder().buildConfig0([:], [file1]) + def cfg1 = new ConfigBuilder().build([file1]) then: cfg1.process.cpus == 1 cfg1.process.ext.args == 'Hello World!' @@ -2149,7 +554,7 @@ class ConfigBuilderTest extends Specification { """ when: - def cfg = new ConfigBuilder().setProfile('foo').buildConfig0([:], [file1]) + def cfg = new ConfigBuilder().setProfile('foo').build([file1]) then: cfg.process == [cpus: 1] cfg.params == [alpha: 1, delta: 20, gamma: 30] @@ -2183,9 +588,8 @@ class ConfigBuilderTest extends Specification { } """ - when: - def cfg = new ConfigBuilder().setProfile('foo').buildConfig0([:], [file1]) + def cfg = new ConfigBuilder().setProfile('foo').build([file1]) then: cfg.process == [cpus: 1] cfg.params == [alpha: 1, delta: 20, gamma: 30] @@ -2221,9 +625,8 @@ class ConfigBuilderTest extends Specification { } ''' - when: - def cfg = new ConfigBuilder().setProfile('foo,bar') .buildConfig0([:], [file1]) + def cfg = new ConfigBuilder().setProfile('foo,bar') .build([file1]) then: cfg.process.cpus == 1 cfg.params.alpha == 1 @@ -2234,232 +637,6 @@ class ConfigBuilderTest extends Specification { folder?.deleteDir() } - def 'CLI params should overwrite only the key provided when nested'() { - given: - def folder = File.createTempDir() - def configMain = new File(folder, 'nextflow.config').absoluteFile - - configMain.text = """ - params { - foo = 'Hello' - bar = "Monde" - baz { - x = "Ciao" - y = "mundo" - z { - alpha = "Hallo" - beta = "World" - } - } - - } - """ - - when: - def opt = new CliOptions() - def run = new CmdRun(params: [bar: "world", 'baz.y': "mondo", 'baz.z.beta': "Welt"]) - def config = new ConfigBuilder(env: [NXF_CONFIG_FILE: configMain.toString()]).setOptions(opt).setCmdRun(run).build() - - then: - config.params.foo == 'Hello' - config.params.bar == 'world' - config.params.baz.x == 'Ciao' - config.params.baz.y == 'mondo' - //tests recursion - config.params.baz.z.alpha == 'Hallo' - config.params.baz.z.beta == 'Welt' - - cleanup: - folder?.deleteDir() - } - - @Unroll - def 'should merge config params' () { - given: - def builder = new ConfigBuilder() - - expect: - def cfg = new ConfigObject(); if(CONFIG) cfg.putAll(CONFIG) - and: - builder.mergeMaps(cfg, PARAMS, false) == EXPECTED - - where: - CONFIG | PARAMS | EXPECTED - [foo:1] | null | [foo:1] - null | [bar:2] | [bar:2] - [foo:1] | [bar:2] | [foo: 1, bar: 2] - [foo:1] | [bar:null] | [foo: 1, bar: null] - [foo:1] | [foo:null] | [foo: null] - [foo:1, bar:[:]] | [bar: [x:10, y:20]] | [foo: 1, bar: [x:10, y:20]] - [foo:1, bar:[x:1, y:2]] | [bar: [x:10, y:20]] | [foo: 1, bar: [x:10, y:20]] - [foo:1, bar:[x:1, y:2]] | [foo: 2, bar: [x:10, y:20]] | [foo: 2, bar: [x:10, y:20]] - [foo:1, bar:null] | [bar: [x:10, y:20]] | [foo: 1, bar: [x:10, y:20]] - [foo:1, bar:2] | [bar: [x:10, y:20]] | [foo: 1, bar: [x:10, y:20]] - [foo:1, bar:[x:1, y:2]] | [bar: 2] | [foo: 1, bar: 2] - } - - @Unroll - def 'should merge config strict params' () { - given: - def builder = new ConfigBuilder() - - expect: - def cfg = new ConfigObject(); if(CONFIG) cfg.putAll(CONFIG) - and: - builder.mergeMaps(cfg, PARAMS, true) == EXPECTED - - where: - CONFIG | PARAMS | EXPECTED - [:] | [bar:2] | [bar:2] - [foo:1] | null | [foo:1] - null | [bar:2] | [bar:2] - [foo:1] | [bar:2] | [foo: 1, bar: 2] - [foo:1] | [bar:null] | [foo: 1, bar: null] - [foo:1] | [foo:null] | [foo: null] - [foo:1, bar:[:]] | [bar: [x:10, y:20]] | [foo: 1, bar: [x:10, y:20]] - [foo:1, bar:[x:1, y:2]] | [bar: [x:10, y:20]] | [foo: 1, bar: [x:10, y:20]] - [foo:1, bar:[x:1, y:2]] | [foo: 2, bar: [x:10, y:20]] | [foo: 2, bar: [x:10, y:20]] - } - - def 'prevent config side effects' () { - given: - def folder = Files.createTempDirectory('test') - and: - def config = folder.resolve('nf.config') - config.text = '''\ - params.test.foo = "foo_def" - params.test.bar = "bar_def" - '''.stripIndent() - - when: - def cfg1 = new ConfigBuilder() - .setOptions( new CliOptions(userConfig: [config.toString()])) - .build() - then: - cfg1.params.test.foo == "foo_def" - cfg1.params.test.bar == "bar_def" - - - when: - def cfg2 = new ConfigBuilder() - .setOptions( new CliOptions(userConfig: [config.toString()])) - .setCmdRun( new CmdRun(params: ['test.foo': 'CLI_FOO'] )) - .build() - then: - cfg2.params.test.foo == "CLI_FOO" - cfg2.params.test.bar == "bar_def" - - cleanup: - folder?.deleteDir() - } - - def 'parse nested json' () { - given: - def folder = Files.createTempDirectory('test') - and: - def config = folder.resolve('nf.json') - config.text = '''\ - { - "title": "something", - "nested": { - "name": "Mike", - "and": { - "more": "nesting", - "still": { - "another": "layer" - } - } - } - } - '''.stripIndent() - - when: - def cfg1 = new ConfigBuilder().setCmdRun(new CmdRun(paramsFile: config.toString())).build() - - then: - cfg1.params.title == "something" - cfg1.params.nested.name == 'Mike' - cfg1.params.nested.and.more == 'nesting' - cfg1.params.nested.and.still.another == 'layer' - - cleanup: - folder?.deleteDir() - } - - def 'parse nested yaml' () { - given: - def folder = Files.createTempDirectory('test') - and: - def config = folder.resolve('nf.yaml') - config.text = '''\ - title: "something" - nested: - name: "Mike" - and: - more: nesting - still: - another: layer - '''.stripIndent() - - when: - def cfg1 = new ConfigBuilder().setCmdRun(new CmdRun(paramsFile: config.toString())).build() - - then: - cfg1.params.title == "something" - cfg1.params.nested.name == 'Mike' - cfg1.params.nested.and.more == 'nesting' - cfg1.params.nested.and.still.another == 'layer' - - cleanup: - folder?.deleteDir() - } - - - def 'should return parsed config' () { - given: - def cmd = new CmdRun(profile: 'first', withTower: 'http://foo.com', launcher: new Launcher()) - def base = Files.createTempDirectory('test') - base.resolve('nextflow.config').text = ''' - profiles { - first { - params { - foo = 'Hello world' - awsKey = 'xyz' - } - process { - executor = { 'local' } - } - } - second { - params.none = 'Blah' - } - } - ''' - when: - def txt = ConfigBuilder.resolveConfig(base, cmd) - then: - txt == '''\ - params { - foo = 'Hello world' - awsKey = '[secret]' - } - - process { - executor = { 'local' } - } - - workDir = 'work' - - tower { - enabled = true - endpoint = 'http://foo.com' - } - '''.stripIndent() - - cleanup: - base?.deleteDir() - } - def 'should merge profiles with conditions' () { given: def folder = Files.createTempDirectory("mergeprofiles") @@ -2499,7 +676,7 @@ class ConfigBuilderTest extends Specification { ''' when: - def cfg = new ConfigBuilder().setProfile('test').buildConfig0([:], [main]) + def cfg = new ConfigBuilder().setProfile('test').build([main]) then: cfg.process.'withName:FOO' cfg.params.load_config == true @@ -2510,7 +687,6 @@ class ConfigBuilderTest extends Specification { folder?.deleteDir() } - def 'should build config object with secrets' () { given: SecretsLoader.instance.reset() @@ -2536,7 +712,7 @@ class ConfigBuilderTest extends Specification { ''' when: - def cfg = new ConfigBuilder().setBaseDir(folder).buildConfig0([:], [text]) + def cfg = new ConfigBuilder().setBaseDir(folder).build([text]) then: cfg.params.p == "$folder/1" cfg.params.s == 'ciao/2' @@ -2594,8 +770,7 @@ class ConfigBuilderTest extends Specification { ''' when: - def opt = new CliOptions() - def config = new ConfigBuilder().setBaseDir(folder).setOptions(opt).buildGivenFiles(configMain) + def config = new ConfigBuilder().setBaseDir(folder).build([configMain]) then: config.p1 == 'one' config.p2 == 'two' @@ -2651,8 +826,7 @@ class ConfigBuilderTest extends Specification { ''' when: - def opt = new CliOptions() - def config = new ConfigBuilder().setBaseDir(folder).setOptions(opt).buildGivenFiles(configMain) + def config = new ConfigBuilder().setBaseDir(folder).build([configMain]) then: config.p1 == 'one' config.p2 == 'two' @@ -2731,12 +905,10 @@ class ConfigBuilderTest extends Specification { ''' when: - def opt = new CliOptions() def config = new ConfigBuilder() - .setBaseDir(folder) - .setOptions(opt) - .setStripSecrets(true) - .buildGivenFiles(configMain) + .setBaseDir(folder) + .setStripSecrets(true) + .build([configMain]) then: config.p1 == 'secrets.ALPHA' config.p2 == 'secrets.DELTA' @@ -2756,4 +928,3 @@ class ConfigBuilderTest extends Specification { } } - diff --git a/modules/nextflow/src/test/groovy/nextflow/config/parser/v1/ConfigParserV1Test.groovy b/modules/nextflow/src/test/groovy/nextflow/config/parser/v1/ConfigParserV1Test.groovy index 43fbf761cd..a8e9e10b5b 100644 --- a/modules/nextflow/src/test/groovy/nextflow/config/parser/v1/ConfigParserV1Test.groovy +++ b/modules/nextflow/src/test/groovy/nextflow/config/parser/v1/ConfigParserV1Test.groovy @@ -566,7 +566,7 @@ class ConfigParserV1Test extends Specification { when: def url = 'http://localhost:9900/nextflow.config' as Path - def cfg = new ConfigBuilder().buildGivenFiles(url) + def cfg = new ConfigBuilder().build([url]) then: cfg.params.foo == 'Hello' cfg.params.bar == 'world!' diff --git a/modules/nextflow/src/test/groovy/nextflow/config/parser/v2/ConfigParserV2Test.groovy b/modules/nextflow/src/test/groovy/nextflow/config/parser/v2/ConfigParserV2Test.groovy index e4843d6205..3922494041 100644 --- a/modules/nextflow/src/test/groovy/nextflow/config/parser/v2/ConfigParserV2Test.groovy +++ b/modules/nextflow/src/test/groovy/nextflow/config/parser/v2/ConfigParserV2Test.groovy @@ -510,7 +510,7 @@ class ConfigParserV2Test extends Specification { when: def url = 'http://localhost:9900/nextflow.config' as Path - def cfg = new ConfigBuilder().buildGivenFiles(url) + def cfg = new ConfigBuilder().build([url]) then: cfg.params.foo == 'Hello' cfg.params.bar == 'world!' diff --git a/modules/nextflow/src/test/groovy/nextflow/scm/ProviderPathTest.groovy b/modules/nextflow/src/test/groovy/nextflow/scm/ProviderPathTest.groovy index 4a6132a288..8b6921c585 100644 --- a/modules/nextflow/src/test/groovy/nextflow/scm/ProviderPathTest.groovy +++ b/modules/nextflow/src/test/groovy/nextflow/scm/ProviderPathTest.groovy @@ -46,7 +46,7 @@ class ProviderPathTest extends Specification { ''' when: - def cfg = new ConfigBuilder().buildGivenFiles(path) + def cfg = new ConfigBuilder().build([path]) then: provider.readText('nextflow.config') >> MAIN_CONFIG provider.readText('conf/nested.config') >> NESTED_CONFIG diff --git a/modules/nextflow/src/test/groovy/nextflow/util/LoggerHelperTest.groovy b/modules/nextflow/src/test/groovy/nextflow/util/LoggerHelperTest.groovy index 0ad318f117..ec38b500d8 100644 --- a/modules/nextflow/src/test/groovy/nextflow/util/LoggerHelperTest.groovy +++ b/modules/nextflow/src/test/groovy/nextflow/util/LoggerHelperTest.groovy @@ -22,7 +22,6 @@ import java.nio.file.Paths import ch.qos.logback.classic.Level import groovyx.gpars.dataflow.DataflowQueue import groovyx.gpars.dataflow.DataflowVariable -import nextflow.cli.CliOptions import nextflow.extension.OpCall import nextflow.script.BaseScript import nextflow.script.ScriptBinding @@ -64,7 +63,7 @@ class LoggerHelperTest extends Specification { def 'should create LoggerHelper object' () { given: - def logger = new LoggerHelper(Mock(CliOptions)) + def logger = new LoggerHelper(Mock(LoggerOptions)) when: logger.setDaemon(true) then: diff --git a/modules/nf-cli-v1/build.gradle b/modules/nf-cli-v1/build.gradle new file mode 100644 index 0000000000..b8354292e4 --- /dev/null +++ b/modules/nf-cli-v1/build.gradle @@ -0,0 +1,63 @@ +plugins { + id "com.gradleup.shadow" version "8.3.5" +} +apply plugin: 'groovy' +apply plugin: 'application' + +sourceSets { + main.java.srcDirs = [] + main.groovy.srcDirs = ['src/main/java', 'src/main/groovy'] + main.resources.srcDirs = ['src/main/resources'] + test.java.srcDirs = [] + test.groovy.srcDirs = ['src/test/groovy'] + test.resources.srcDirs = ['src/test/resources'] +} + +compileGroovy { + options.compilerArgs = ['-XDignore.symbol.file'] +} + +configurations { + lineageImplementation +} + +dependencies { + api(project(':nextflow')) + api "com.beust:jcommander:1.35" + // note: declare as separate dependency to avoid a circular dependency + lineageImplementation (project(':nf-lineage')) + + testImplementation 'org.subethamail:subethasmtp:3.1.7' + + // test configuration + testImplementation(testFixtures(project(":nextflow"))) + testFixturesImplementation(project(":nextflow")) +} + +test { + minHeapSize = "512m" + maxHeapSize = "4096m" +} + +application { + mainClass = 'nextflow.cli.Launcher' +} + +run { + args( (project.hasProperty("runCmd") ? project.findProperty("runCmd") : "set a cmd to run").split(' ') ) +} + +shadowJar { + // add 'lineage' because it cannot be added to this project + // explicitly otherwise it will result into a circular dependency + configurations = [project.configurations.runtimeClasspath, project.configurations.lineageImplementation] + archiveClassifier='one' + manifest { + attributes 'Main-Class': "$mainClassName" + } + mergeServiceFiles() + mergeGroovyExtensionModules() + transform(com.github.jengelman.gradle.plugins.shadow.transformers.AppendingTransformer) { + resource = 'META-INF/extensions.idx' + } +} diff --git a/modules/nextflow/src/main/groovy/nextflow/cli/CacheBase.groovy b/modules/nf-cli-v1/src/main/groovy/nextflow/cli/CacheBase.groovy similarity index 100% rename from modules/nextflow/src/main/groovy/nextflow/cli/CacheBase.groovy rename to modules/nf-cli-v1/src/main/groovy/nextflow/cli/CacheBase.groovy diff --git a/modules/nextflow/src/main/groovy/nextflow/cli/CliOptions.groovy b/modules/nf-cli-v1/src/main/groovy/nextflow/cli/CliOptions.groovy similarity index 100% rename from modules/nextflow/src/main/groovy/nextflow/cli/CliOptions.groovy rename to modules/nf-cli-v1/src/main/groovy/nextflow/cli/CliOptions.groovy diff --git a/modules/nextflow/src/main/groovy/nextflow/cli/CmdBase.groovy b/modules/nf-cli-v1/src/main/groovy/nextflow/cli/CmdBase.groovy similarity index 100% rename from modules/nextflow/src/main/groovy/nextflow/cli/CmdBase.groovy rename to modules/nf-cli-v1/src/main/groovy/nextflow/cli/CmdBase.groovy diff --git a/modules/nextflow/src/main/groovy/nextflow/cli/CmdClean.groovy b/modules/nf-cli-v1/src/main/groovy/nextflow/cli/CmdClean.groovy similarity index 97% rename from modules/nextflow/src/main/groovy/nextflow/cli/CmdClean.groovy rename to modules/nf-cli-v1/src/main/groovy/nextflow/cli/CmdClean.groovy index bb58edacbb..efd4b98708 100644 --- a/modules/nextflow/src/main/groovy/nextflow/cli/CmdClean.groovy +++ b/modules/nf-cli-v1/src/main/groovy/nextflow/cli/CmdClean.groovy @@ -20,7 +20,6 @@ import java.nio.file.FileVisitor import java.nio.file.Files import java.nio.file.NoSuchFileException import java.nio.file.Path -import java.nio.file.Paths import java.nio.file.attribute.BasicFileAttributes import com.beust.jcommander.Parameter @@ -33,6 +32,7 @@ import nextflow.Global import nextflow.ISession import nextflow.Session import nextflow.config.ConfigBuilder +import nextflow.config.ConfigCmdAdapter import nextflow.exception.AbortOperationException import nextflow.file.FileHelper import nextflow.plugin.Plugins @@ -107,11 +107,11 @@ class CmdClean extends CmdBase implements CacheBase { Global.setLazySession { final builder = new ConfigBuilder() .setShowClosures(true) - .showMissingVariables(true) + .setShowMissingVariables(true) + final config = new ConfigCmdAdapter(builder) .setOptions(launcher.options) - .setBaseDir(Paths.get('.')) - - final config = builder.buildConfigObject() + .setBaseDir(Path.of('.')) + .buildConfigObject() return (ISession) new Session(config) } } diff --git a/modules/nextflow/src/main/groovy/nextflow/cli/CmdClone.groovy b/modules/nf-cli-v1/src/main/groovy/nextflow/cli/CmdClone.groovy similarity index 95% rename from modules/nextflow/src/main/groovy/nextflow/cli/CmdClone.groovy rename to modules/nf-cli-v1/src/main/groovy/nextflow/cli/CmdClone.groovy index 9c582aaaf6..7d08b487d6 100644 --- a/modules/nextflow/src/main/groovy/nextflow/cli/CmdClone.groovy +++ b/modules/nf-cli-v1/src/main/groovy/nextflow/cli/CmdClone.groovy @@ -30,7 +30,7 @@ import nextflow.scm.AssetManager @Slf4j @CompileStatic @Parameters(commandDescription = "Clone a project into a folder") -class CmdClone extends CmdBase implements HubOptions { +class CmdClone extends CmdBase implements HubAware { static final public NAME = 'clone' @@ -52,7 +52,7 @@ class CmdClone extends CmdBase implements HubOptions { Plugins.init() // the pipeline name String pipeline = args[0] - final manager = new AssetManager(pipeline, this) + final manager = new AssetManager(pipeline, toHubOptions()) // the target directory is the second parameter // otherwise default the current pipeline name diff --git a/modules/nextflow/src/main/groovy/nextflow/cli/CmdConfig.groovy b/modules/nf-cli-v1/src/main/groovy/nextflow/cli/CmdConfig.groovy similarity index 97% rename from modules/nextflow/src/main/groovy/nextflow/cli/CmdConfig.groovy rename to modules/nf-cli-v1/src/main/groovy/nextflow/cli/CmdConfig.groovy index c4fe0f681b..db178aca16 100644 --- a/modules/nextflow/src/main/groovy/nextflow/cli/CmdConfig.groovy +++ b/modules/nf-cli-v1/src/main/groovy/nextflow/cli/CmdConfig.groovy @@ -26,6 +26,7 @@ import groovy.transform.PackageScope import groovy.util.logging.Slf4j import nextflow.NF import nextflow.config.ConfigBuilder +import nextflow.config.ConfigCmdAdapter import nextflow.config.ConfigValidator import nextflow.exception.AbortOperationException import nextflow.plugin.Plugins @@ -110,12 +111,13 @@ class CmdConfig extends CmdBase { final builder = new ConfigBuilder() .setShowClosures(true) .setStripSecrets(true) - .showMissingVariables(true) + .setShowMissingVariables(true) + + final config = new ConfigCmdAdapter(builder) .setOptions(launcher.options) .setBaseDir(base) .setCmdConfig(this) - - final config = builder.buildConfigObject() + .buildConfigObject() // -- validate config options if( NF.isSyntaxParserV2() ) { diff --git a/modules/nextflow/src/main/groovy/nextflow/cli/CmdConsole.groovy b/modules/nf-cli-v1/src/main/groovy/nextflow/cli/CmdConsole.groovy similarity index 100% rename from modules/nextflow/src/main/groovy/nextflow/cli/CmdConsole.groovy rename to modules/nf-cli-v1/src/main/groovy/nextflow/cli/CmdConsole.groovy diff --git a/modules/nextflow/src/main/groovy/nextflow/cli/CmdDrop.groovy b/modules/nf-cli-v1/src/main/groovy/nextflow/cli/CmdDrop.groovy similarity index 100% rename from modules/nextflow/src/main/groovy/nextflow/cli/CmdDrop.groovy rename to modules/nf-cli-v1/src/main/groovy/nextflow/cli/CmdDrop.groovy diff --git a/modules/nextflow/src/main/groovy/nextflow/cli/CmdFs.groovy b/modules/nf-cli-v1/src/main/groovy/nextflow/cli/CmdFs.groovy similarity index 98% rename from modules/nextflow/src/main/groovy/nextflow/cli/CmdFs.groovy rename to modules/nf-cli-v1/src/main/groovy/nextflow/cli/CmdFs.groovy index 85abaaaa5e..31ef602960 100644 --- a/modules/nextflow/src/main/groovy/nextflow/cli/CmdFs.groovy +++ b/modules/nf-cli-v1/src/main/groovy/nextflow/cli/CmdFs.groovy @@ -30,7 +30,7 @@ import groovy.transform.CompileStatic import groovy.util.logging.Slf4j import nextflow.Global import nextflow.Session -import nextflow.config.ConfigBuilder +import nextflow.config.ConfigCmdAdapter import nextflow.exception.AbortOperationException import nextflow.extension.FilesEx import nextflow.file.FileHelper @@ -202,7 +202,7 @@ class CmdFs extends CmdBase implements UsageAware { private Session createSession() { // create the config - final config = new ConfigBuilder() + final config = new ConfigCmdAdapter() .setOptions(getLauncher().getOptions()) .setBaseDir(Paths.get('.')) .build() diff --git a/modules/nextflow/src/main/groovy/nextflow/cli/CmdHelp.groovy b/modules/nf-cli-v1/src/main/groovy/nextflow/cli/CmdHelp.groovy similarity index 100% rename from modules/nextflow/src/main/groovy/nextflow/cli/CmdHelp.groovy rename to modules/nf-cli-v1/src/main/groovy/nextflow/cli/CmdHelp.groovy diff --git a/modules/nextflow/src/main/groovy/nextflow/cli/CmdHelper.groovy b/modules/nf-cli-v1/src/main/groovy/nextflow/cli/CmdHelper.groovy similarity index 100% rename from modules/nextflow/src/main/groovy/nextflow/cli/CmdHelper.groovy rename to modules/nf-cli-v1/src/main/groovy/nextflow/cli/CmdHelper.groovy diff --git a/modules/nextflow/src/main/groovy/nextflow/cli/CmdInfo.groovy b/modules/nf-cli-v1/src/main/groovy/nextflow/cli/CmdInfo.groovy similarity index 100% rename from modules/nextflow/src/main/groovy/nextflow/cli/CmdInfo.groovy rename to modules/nf-cli-v1/src/main/groovy/nextflow/cli/CmdInfo.groovy diff --git a/modules/nextflow/src/main/groovy/nextflow/cli/CmdInspect.groovy b/modules/nf-cli-v1/src/main/groovy/nextflow/cli/CmdInspect.groovy similarity index 100% rename from modules/nextflow/src/main/groovy/nextflow/cli/CmdInspect.groovy rename to modules/nf-cli-v1/src/main/groovy/nextflow/cli/CmdInspect.groovy diff --git a/modules/nextflow/src/main/groovy/nextflow/cli/CmdKubeRun.groovy b/modules/nf-cli-v1/src/main/groovy/nextflow/cli/CmdKubeRun.groovy similarity index 100% rename from modules/nextflow/src/main/groovy/nextflow/cli/CmdKubeRun.groovy rename to modules/nf-cli-v1/src/main/groovy/nextflow/cli/CmdKubeRun.groovy diff --git a/modules/nextflow/src/main/groovy/nextflow/cli/CmdLineage.groovy b/modules/nf-cli-v1/src/main/groovy/nextflow/cli/CmdLineage.groovy similarity index 98% rename from modules/nextflow/src/main/groovy/nextflow/cli/CmdLineage.groovy rename to modules/nf-cli-v1/src/main/groovy/nextflow/cli/CmdLineage.groovy index 55f075fdc6..60b3391a5f 100644 --- a/modules/nextflow/src/main/groovy/nextflow/cli/CmdLineage.groovy +++ b/modules/nf-cli-v1/src/main/groovy/nextflow/cli/CmdLineage.groovy @@ -22,7 +22,7 @@ import java.nio.file.Paths import com.beust.jcommander.Parameter import com.beust.jcommander.Parameters import groovy.transform.CompileStatic -import nextflow.config.ConfigBuilder +import nextflow.config.ConfigCmdAdapter import nextflow.config.ConfigMap import nextflow.exception.AbortOperationException import nextflow.plugin.Plugins @@ -85,7 +85,7 @@ class CmdLineage extends CmdBase implements UsageAware { // setup the plugins system and load the secrets provider Plugins.init() // load the config - this.config = new ConfigBuilder() + this.config = new ConfigCmdAdapter() .setOptions(launcher.options) .setBaseDir(Paths.get('.')) .build() diff --git a/modules/nextflow/src/main/groovy/nextflow/cli/CmdLint.groovy b/modules/nf-cli-v1/src/main/groovy/nextflow/cli/CmdLint.groovy similarity index 100% rename from modules/nextflow/src/main/groovy/nextflow/cli/CmdLint.groovy rename to modules/nf-cli-v1/src/main/groovy/nextflow/cli/CmdLint.groovy diff --git a/modules/nextflow/src/main/groovy/nextflow/cli/CmdList.groovy b/modules/nf-cli-v1/src/main/groovy/nextflow/cli/CmdList.groovy similarity index 100% rename from modules/nextflow/src/main/groovy/nextflow/cli/CmdList.groovy rename to modules/nf-cli-v1/src/main/groovy/nextflow/cli/CmdList.groovy diff --git a/modules/nextflow/src/main/groovy/nextflow/cli/CmdLog.groovy b/modules/nf-cli-v1/src/main/groovy/nextflow/cli/CmdLog.groovy similarity index 100% rename from modules/nextflow/src/main/groovy/nextflow/cli/CmdLog.groovy rename to modules/nf-cli-v1/src/main/groovy/nextflow/cli/CmdLog.groovy diff --git a/modules/nextflow/src/main/groovy/nextflow/cli/CmdNode.groovy b/modules/nf-cli-v1/src/main/groovy/nextflow/cli/CmdNode.groovy similarity index 95% rename from modules/nextflow/src/main/groovy/nextflow/cli/CmdNode.groovy rename to modules/nf-cli-v1/src/main/groovy/nextflow/cli/CmdNode.groovy index 47666166d6..bda0f58197 100644 --- a/modules/nextflow/src/main/groovy/nextflow/cli/CmdNode.groovy +++ b/modules/nf-cli-v1/src/main/groovy/nextflow/cli/CmdNode.groovy @@ -20,7 +20,7 @@ import com.beust.jcommander.Parameter import com.beust.jcommander.Parameters import groovy.transform.CompileStatic import groovy.util.logging.Slf4j -import nextflow.config.ConfigBuilder +import nextflow.config.ConfigCmdAdapter import nextflow.daemon.DaemonLauncher import nextflow.plugin.Plugins import nextflow.util.ServiceName @@ -65,10 +65,10 @@ class CmdNode extends CmdBase { protected launchDaemon(String name = null) { // create the config object - def config = new ConfigBuilder() - .setOptions(launcher.options) - .setCmdNode(this) - .build() + final config = new ConfigCmdAdapter() + .setOptions(launcher.options) + .setCmdNode(this) + .build() DaemonLauncher instance if( name ) { diff --git a/modules/nextflow/src/main/groovy/nextflow/cli/CmdPlugin.groovy b/modules/nf-cli-v1/src/main/groovy/nextflow/cli/CmdPlugin.groovy similarity index 100% rename from modules/nextflow/src/main/groovy/nextflow/cli/CmdPlugin.groovy rename to modules/nf-cli-v1/src/main/groovy/nextflow/cli/CmdPlugin.groovy diff --git a/modules/nextflow/src/main/groovy/nextflow/cli/CmdPull.groovy b/modules/nf-cli-v1/src/main/groovy/nextflow/cli/CmdPull.groovy similarity index 95% rename from modules/nextflow/src/main/groovy/nextflow/cli/CmdPull.groovy rename to modules/nf-cli-v1/src/main/groovy/nextflow/cli/CmdPull.groovy index 0c99430964..cfa0fba935 100644 --- a/modules/nextflow/src/main/groovy/nextflow/cli/CmdPull.groovy +++ b/modules/nf-cli-v1/src/main/groovy/nextflow/cli/CmdPull.groovy @@ -31,7 +31,7 @@ import nextflow.util.TestOnly @Slf4j @CompileStatic @Parameters(commandDescription = "Download or update a project") -class CmdPull extends CmdBase implements HubOptions { +class CmdPull extends CmdBase implements HubAware { static final public NAME = 'pull' @@ -74,7 +74,7 @@ class CmdPull extends CmdBase implements HubOptions { list.each { log.info "Checking $it ..." - def manager = new AssetManager(it, this) + def manager = new AssetManager(it, toHubOptions()) def result = manager.download(revision,deep) manager.updateModules() diff --git a/modules/nextflow/src/main/groovy/nextflow/cli/CmdRun.groovy b/modules/nf-cli-v1/src/main/groovy/nextflow/cli/CmdRun.groovy similarity index 98% rename from modules/nextflow/src/main/groovy/nextflow/cli/CmdRun.groovy rename to modules/nf-cli-v1/src/main/groovy/nextflow/cli/CmdRun.groovy index d096e678e6..4c40f44e77 100644 --- a/modules/nextflow/src/main/groovy/nextflow/cli/CmdRun.groovy +++ b/modules/nf-cli-v1/src/main/groovy/nextflow/cli/CmdRun.groovy @@ -36,6 +36,7 @@ import nextflow.NF import nextflow.NextflowMeta import nextflow.SysEnv import nextflow.config.ConfigBuilder +import nextflow.config.ConfigCmdAdapter import nextflow.config.ConfigMap import nextflow.config.ConfigValidator import nextflow.exception.AbortOperationException @@ -58,7 +59,7 @@ import org.yaml.snakeyaml.Yaml @Slf4j @CompileStatic @Parameters(commandDescription = "Execute a pipeline project") -class CmdRun extends CmdBase implements HubOptions { +class CmdRun extends CmdBase implements HubAware { static final public Pattern RUN_NAME_PATTERN = Pattern.compile(/^[a-z](?:[a-z\d]|[-_](?=[a-z\d])){0,79}$/, Pattern.CASE_INSENSITIVE) @@ -327,10 +328,11 @@ class CmdRun extends CmdBase implements HubOptions { // create the config object final builder = new ConfigBuilder() + final adapter = new ConfigCmdAdapter(builder) .setOptions(launcher.options) .setCmdRun(this) .setBaseDir(scriptFile.parent) - final config = builder .build() + final config = adapter.build() // check DSL syntax in the config launchInfo(config, scriptFile) @@ -354,9 +356,9 @@ class CmdRun extends CmdBase implements HubOptions { runner.session.disableJobsCancellation = getDisableJobsCancellation() final isTowerEnabled = config.navigate('tower.enabled') as Boolean - final isDataEnabled = config.navigate("lineage.enabled") as Boolean - if( isTowerEnabled || isDataEnabled || log.isTraceEnabled() ) - runner.session.resolvedConfig = ConfigBuilder.resolveConfig(scriptFile.parent, this) + final isLineageEnabled = config.navigate("lineage.enabled") as Boolean + if( isTowerEnabled || isLineageEnabled || log.isTraceEnabled() ) + runner.session.resolvedConfig = ConfigCmdAdapter.resolveConfig(scriptFile.parent, this) // note config files are collected during the build process // this line should be after `ConfigBuilder#build` runner.session.configFiles = builder.parsedConfigFiles @@ -580,7 +582,7 @@ class CmdRun extends CmdBase implements HubOptions { /* * try to look for a pipeline in the repository */ - def manager = new AssetManager(pipelineName, this) + def manager = new AssetManager(pipelineName, toHubOptions()) def repo = manager.getProject() boolean checkForUpdate = true diff --git a/modules/nextflow/src/main/groovy/nextflow/cli/CmdSecret.groovy b/modules/nf-cli-v1/src/main/groovy/nextflow/cli/CmdSecret.groovy similarity index 100% rename from modules/nextflow/src/main/groovy/nextflow/cli/CmdSecret.groovy rename to modules/nf-cli-v1/src/main/groovy/nextflow/cli/CmdSecret.groovy diff --git a/modules/nextflow/src/main/groovy/nextflow/cli/CmdSelfUpdate.groovy b/modules/nf-cli-v1/src/main/groovy/nextflow/cli/CmdSelfUpdate.groovy similarity index 100% rename from modules/nextflow/src/main/groovy/nextflow/cli/CmdSelfUpdate.groovy rename to modules/nf-cli-v1/src/main/groovy/nextflow/cli/CmdSelfUpdate.groovy diff --git a/modules/nextflow/src/main/groovy/nextflow/cli/CmdView.groovy b/modules/nf-cli-v1/src/main/groovy/nextflow/cli/CmdView.groovy similarity index 100% rename from modules/nextflow/src/main/groovy/nextflow/cli/CmdView.groovy rename to modules/nf-cli-v1/src/main/groovy/nextflow/cli/CmdView.groovy diff --git a/modules/nextflow/src/main/groovy/nextflow/cli/HubOptions.groovy b/modules/nf-cli-v1/src/main/groovy/nextflow/cli/HubAware.groovy similarity index 51% rename from modules/nextflow/src/main/groovy/nextflow/cli/HubOptions.groovy rename to modules/nf-cli-v1/src/main/groovy/nextflow/cli/HubAware.groovy index 9a022afda1..b87387ea60 100644 --- a/modules/nextflow/src/main/groovy/nextflow/cli/HubOptions.groovy +++ b/modules/nf-cli-v1/src/main/groovy/nextflow/cli/HubAware.groovy @@ -18,14 +18,14 @@ package nextflow.cli import com.beust.jcommander.Parameter import groovy.transform.CompileStatic +import nextflow.scm.HubOptions /** - * Defines the command line parameters for command that need to interact with a pipeline service hub i.e. GitHub or BitBucket + * Command line parameters for commands that interact with a git repository provider i.e. GitHub or BitBucket * * @author Paolo Di Tommaso */ - @CompileStatic -trait HubOptions { +interface HubAware { @Parameter(names=['-hub'], description = "Service hub where the project is hosted") String hubProvider @@ -33,36 +33,7 @@ trait HubOptions { @Parameter(names='-user', description = 'Private repository user name') String hubUser - /** - * Return the password provided on the command line or stop allowing the user to enter it on the console - * - * @return The password entered or {@code null} if no user has been entered - */ - String getHubPassword() { - - if( !hubUser ) - return null - - def p = hubUser.indexOf(':') - if( p != -1 ) - return hubUser.substring(p+1) - - def console = System.console() - if( !console ) - return null - - print "Enter your $hubProvider password: " - char[] pwd = console.readPassword() - new String(pwd) + default HubOptions toHubOptions() { + return new HubOptions(hubProvider, hubUser) } - - String getHubUser() { - if(!hubUser) { - return hubUser - } - - def p = hubUser.indexOf(':') - return p != -1 ? hubUser.substring(0,p) : hubUser - } - } diff --git a/modules/nextflow/src/main/groovy/nextflow/cli/Launcher.groovy b/modules/nf-cli-v1/src/main/groovy/nextflow/cli/Launcher.groovy similarity index 98% rename from modules/nextflow/src/main/groovy/nextflow/cli/Launcher.groovy rename to modules/nf-cli-v1/src/main/groovy/nextflow/cli/Launcher.groovy index 6afda06942..94d04c3858 100644 --- a/modules/nextflow/src/main/groovy/nextflow/cli/Launcher.groovy +++ b/modules/nf-cli-v1/src/main/groovy/nextflow/cli/Launcher.groovy @@ -39,8 +39,8 @@ import nextflow.exception.ScriptRuntimeException import nextflow.secret.SecretsLoader import nextflow.util.Escape import nextflow.util.LoggerHelper +import nextflow.util.LoggerOptions import nextflow.util.ProxyConfig -import nextflow.util.SpuriousDeps import org.eclipse.jgit.api.errors.GitAPIException import static nextflow.util.SysHelper.dumpThreads @@ -115,11 +115,6 @@ class Launcher { if(SecretsLoader.isEnabled()) allCommands.add(new CmdSecret()) - // legacy command - final cmdCloud = SpuriousDeps.cmdCloud() - if( cmdCloud ) - allCommands.add(cmdCloud) - options = new CliOptions() jcommander = new JCommander(options) for( CmdBase cmd : allCommands ) { @@ -443,7 +438,7 @@ class Launcher { */ try { parseMainArgs(args) - LoggerHelper.configureLogger(this) + LoggerHelper.configureLogger(loggerOptions(), daemonMode) } catch( ParameterException e ) { // print command line parsing errors @@ -465,6 +460,18 @@ class Launcher { return this } + protected LoggerOptions loggerOptions() { + new LoggerOptions( + options.ansiLog, + options.background, + options.debug, + options.logFile, + options.quiet, + options.syslog, + options.trace + ) + } + protected void checkForHelp() { if( options.help || !command || command.help ) { if( command instanceof UsageAware ) { diff --git a/modules/nextflow/src/main/groovy/nextflow/cli/PluginAbstractExec.groovy b/modules/nf-cli-v1/src/main/groovy/nextflow/cli/PluginAbstractExec.groovy similarity index 96% rename from modules/nextflow/src/main/groovy/nextflow/cli/PluginAbstractExec.groovy rename to modules/nf-cli-v1/src/main/groovy/nextflow/cli/PluginAbstractExec.groovy index 8cb1391b00..b64cf9a6e9 100644 --- a/modules/nextflow/src/main/groovy/nextflow/cli/PluginAbstractExec.groovy +++ b/modules/nf-cli-v1/src/main/groovy/nextflow/cli/PluginAbstractExec.groovy @@ -21,7 +21,7 @@ import java.nio.file.Paths import groovy.transform.CompileStatic import nextflow.Session -import nextflow.config.ConfigBuilder +import nextflow.config.ConfigCmdAdapter import nextflow.exception.AbortOperationException import org.slf4j.Logger import org.slf4j.LoggerFactory @@ -50,7 +50,7 @@ trait PluginAbstractExec implements PluginExecAware { final int exec(Launcher launcher1, String pluginId, String cmd, List args) { this.launcher = launcher1 // create the config - final config = new ConfigBuilder() + final config = new ConfigCmdAdapter() .setOptions(launcher1.options) .setBaseDir(Paths.get('.')) .build() diff --git a/modules/nextflow/src/main/groovy/nextflow/cli/PluginExecAware.groovy b/modules/nf-cli-v1/src/main/groovy/nextflow/cli/PluginExecAware.groovy similarity index 100% rename from modules/nextflow/src/main/groovy/nextflow/cli/PluginExecAware.groovy rename to modules/nf-cli-v1/src/main/groovy/nextflow/cli/PluginExecAware.groovy diff --git a/modules/nextflow/src/main/groovy/nextflow/cli/UsageAware.groovy b/modules/nf-cli-v1/src/main/groovy/nextflow/cli/UsageAware.groovy similarity index 100% rename from modules/nextflow/src/main/groovy/nextflow/cli/UsageAware.groovy rename to modules/nf-cli-v1/src/main/groovy/nextflow/cli/UsageAware.groovy diff --git a/modules/nf-cli-v1/src/main/groovy/nextflow/config/ConfigCmdAdapter.groovy b/modules/nf-cli-v1/src/main/groovy/nextflow/config/ConfigCmdAdapter.groovy new file mode 100644 index 0000000000..49a1e2e3e8 --- /dev/null +++ b/modules/nf-cli-v1/src/main/groovy/nextflow/config/ConfigCmdAdapter.groovy @@ -0,0 +1,713 @@ +/* + * Copyright 2013-2024, Seqera Labs + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package nextflow.config + +import java.nio.file.Path + +import groovy.transform.CompileDynamic +import groovy.transform.CompileStatic +import groovy.util.logging.Slf4j +import nextflow.Const +import nextflow.NF +import nextflow.cli.CliOptions +import nextflow.cli.CmdConfig +import nextflow.cli.CmdNode +import nextflow.cli.CmdRun +import nextflow.exception.AbortOperationException +import nextflow.trace.GraphObserver +import nextflow.trace.ReportObserver +import nextflow.trace.TimelineObserver +import nextflow.trace.TraceFileObserver +import nextflow.util.HistoryFile +import nextflow.util.SecretHelper + +import static nextflow.util.ConfigHelper.toCanonicalString +/** + * Adapter for building a Nextflow config with CLI overrides + * + * @author Paolo Di Tommaso + */ +@Slf4j +@CompileStatic +class ConfigCmdAdapter { + + ConfigBuilder builder + + CliOptions options + + CmdRun cmdRun + + CmdNode cmdNode + + Path baseDir + + Path currentDir + + Path homeDir + + List userConfigFiles = [] + + Map sysEnv = new HashMap<>(System.getenv()) + + ConfigCmdAdapter(ConfigBuilder builder) { + this.builder = builder + setHomeDir(Const.APP_HOME_DIR) + setCurrentDir(Path.of('.')) + } + + ConfigCmdAdapter() { + this(new ConfigBuilder()) + } + + ConfigCmdAdapter setOptions(CliOptions options) { + this.options = options + builder.setIgnoreIncludes(options?.ignoreConfigIncludes) + return this + } + + ConfigCmdAdapter setCmdConfig(CmdConfig cmdConfig) { + builder.setShowAllProfiles(cmdConfig.showAllProfiles) + builder.setProfile(cmdConfig.profile) + return this + } + + ConfigCmdAdapter setCmdNode(CmdNode cmdNode) { + this.cmdNode = cmdNode + return this + } + + ConfigCmdAdapter setCmdRun(CmdRun cmdRun) { + this.cmdRun = cmdRun + builder.setProfile(cmdRun.profile) + return this + } + + ConfigCmdAdapter setBaseDir(Path path) { + this.baseDir = path.complete() + builder.setBaseDir(baseDir) + return this + } + + ConfigCmdAdapter setCurrentDir(Path path) { + this.currentDir = path.complete() + builder.setCurrentDir(currentDir) + return this + } + + ConfigCmdAdapter setHomeDir(Path path) { + this.homeDir = path.complete() + return this + } + + ConfigCmdAdapter setUserConfigFiles(Path... files) { + setUserConfigFiles(files as List) + return this + } + + ConfigCmdAdapter setUserConfigFiles(List files) { + userConfigFiles.addAll(files) + return this + } + + /** + * Build the configuration as a ConfigMap. + */ + ConfigMap build() { + toConfigMap(buildConfigObject()) + } + + /** + * Build the configuration as a ConfigObject. + */ + ConfigObject buildConfigObject() { + // -- configuration file(s) + final configFiles = resolveConfigFiles(options?.config) + validateConfigFiles(configFiles) + final config = buildGivenFiles(configFiles) + + if( cmdRun ) + configRunOptions(config, System.getenv(), cmdRun) + + return config + } + + /** + * Transform the specified list of string to a list of files. + * + * The following default configuration files are used if no files are specified: + * - $HOME/.nextflow/config + * - $PWD/nextflow.config + * + * @param files + * @return + */ + protected List resolveConfigFiles(List files) { + + if( files ) { + return files.stream() + .map(filename -> currentDir.resolve(filename)) + .toList() + } + + final List result = [] + + /* + * config file in the nextflow home + */ + final home = homeDir.resolve('config') + if( home.exists() ) { + log.debug "Found config home: $home" + result << home + } + + /** + * Config file in the pipeline base dir + * This config file name should be predictable, therefore cannot be overridden + */ + def base = null + if( baseDir && baseDir != currentDir ) { + base = baseDir.resolve('nextflow.config') + if( base.exists() ) { + log.debug "Found config base: $base" + result << base + } + } + + /** + * Local or user provided file + * Default config file name can be overridden with `NXF_CONFIG_FILE` env variable + */ + final configFileName = sysEnv.get('NXF_CONFIG_FILE') ?: 'nextflow.config' + final local = currentDir.resolve(configFileName) + if( local.exists() && local != base ) { + log.debug "Found config local: $local" + result << local + } + + final customConfigs = [] + if( userConfigFiles ) + customConfigs.addAll(userConfigFiles) + if( options?.userConfig ) + customConfigs.addAll(options.userConfig) + if( cmdRun?.runConfig ) + customConfigs.addAll(cmdRun.runConfig) + for( final item : customConfigs ) { + final configFile = item instanceof Path ? item : currentDir.resolve(item.toString()) + log.debug "User config file: $configFile" + result << configFile + } + + return result + } + + /** + * Validate the existence of a list of config files. + * + * @param files + */ + protected void validateConfigFiles(List files) { + for( final file : files ) { + if( !file.exists() ) + throw new AbortOperationException("The specified configuration file does not exist: $file -- check the name or choose another file") + if( !file.isFile() ) + throw new AbortOperationException("The specified configuration file is not a file: $file -- check the name or choose another file") + } + } + + /** + * Create the nextflow configuration {@link ConfigObject} given a one or more + * config files + * + * @param files A list of config files {@link Path} + * @return The resulting {@link ConfigObject} instance + */ + protected ConfigObject buildGivenFiles(List files) { + + if( cmdRun && (cmdRun.hasParams()) ) + builder.setParams(cmdRun.parsedParams(builder.configVars())) + + def items = [] + if( files ) for( Path file : files ) { + log.debug "Parsing config file: ${file.complete()}" + if (!file.exists()) { + log.warn "The specified configuration file cannot be found: $file" + } + else { + items << file + } + } + + final Map env = [:] + if( cmdRun?.exportSysEnv ) { + log.debug "Adding current system environment to session environment" + env.putAll(System.getenv()) + } + + final envVars = cmdRun?.env + if( envVars ) { + log.debug "Adding the following variables to session environment: $envVars" + env.putAll(envVars) + } + + // set the cluster options for the node command + if( cmdNode?.clusterOptions ) { + def str = new StringBuilder() + cmdNode.clusterOptions.each { k, v -> + str << "cluster." << k << '=' << wrapValue(v) << '\n' + } + items << str + } + + // -- add the executor obj from the command line args + if( cmdRun?.clusterOptions ) { + def str = new StringBuilder() + cmdRun.clusterOptions.each { k, v -> + str << "cluster." << k << '=' << wrapValue(v) << '\n' + } + items << str + } + + if( cmdRun?.executorOptions ) { + def str = new StringBuilder() + cmdRun.executorOptions.each { k, v -> + str << "executor." << k << '=' << wrapValue(v) << '\n' + } + items << str + } + + return builder.build(env, items) + } + + protected ConfigObject buildGivenFiles(Path... files) { + return buildGivenFiles(files as List) + } + + private static String wrapValue( value ) { + if( !value ) + return '' + + value = value.toString().trim() + if( value == 'true' || value == 'false') + return value + + if( value.isNumber() ) + return value + + return "'$value'" + } + + /** + * Apply command-line arguments to the config. + * + * @param config + * @param env + * @param cmdRun + */ + @CompileDynamic + protected void configRunOptions(ConfigObject config, Map env, CmdRun cmdRun) { + + // -- set config options + if( cmdRun.cacheable != null ) + config.cacheable = cmdRun.cacheable + + // -- set the run name + if( cmdRun.runName ) + config.runName = cmdRun.runName + + if( cmdRun.stubRun ) + config.stubRun = cmdRun.stubRun + + // -- set the output directory + if( cmdRun.outputDir ) + config.outputDir = cmdRun.outputDir + + if( cmdRun.preview ) + config.preview = cmdRun.preview + + // -- sets the working directory + if( cmdRun.workDir ) + config.workDir = cmdRun.workDir + + else if( !config.workDir ) + config.workDir = env.get('NXF_WORK') ?: 'work' + + if( cmdRun.bucketDir ) + config.bucketDir = cmdRun.bucketDir + + // -- sets the library path + if( cmdRun.libPath ) + config.libDir = cmdRun.libPath + + else if ( !config.isSet('libDir') && env.get('NXF_LIB') ) + config.libDir = env.get('NXF_LIB') + + // -- override 'process' parameters defined on the cmd line + cmdRun.process.each { name, value -> + config.process[name] = parseValue(value) + } + + if( cmdRun.withoutConda && config.conda instanceof Map ) { + // disable conda execution + log.debug "Disabling execution with Conda as requested by command-line option `-without-conda`" + config.conda.enabled = false + } + + // -- apply the conda environment + if( cmdRun.withConda ) { + if( cmdRun.withConda != '-' ) + config.process.conda = cmdRun.withConda + config.conda.enabled = true + } + + if( cmdRun.withoutSpack && config.spack instanceof Map ) { + // disable spack execution + log.debug "Disabling execution with Spack as requested by command-line option `-without-spack`" + config.spack.enabled = false + } + + // -- apply the spack environment + if( cmdRun.withSpack ) { + if( cmdRun.withSpack != '-' ) + config.process.spack = cmdRun.withSpack + config.spack.enabled = true + } + + // -- sets the resume option + if( cmdRun.resume ) + config.resume = cmdRun.resume + + if( config.isSet('resume') ) + config.resume = normalizeResumeId(config.resume as String) + + // -- sets `dumpHashes` option + if( cmdRun.dumpHashes ) { + config.dumpHashes = cmdRun.dumpHashes != '-' ? cmdRun.dumpHashes : 'default' + } + + if( cmdRun.dumpChannels ) + config.dumpChannels = cmdRun.dumpChannels.tokenize(',') + + // -- other configuration parameters + if( cmdRun.poolSize ) { + config.poolSize = cmdRun.poolSize + } + if( cmdRun.queueSize ) { + config.executor.queueSize = cmdRun.queueSize + } + if( cmdRun.pollInterval ) { + config.executor.pollInterval = cmdRun.pollInterval + } + + // -- sets trace file options + if( cmdRun.withTrace ) { + if( !(config.trace instanceof Map) ) + config.trace = [:] + config.trace.enabled = true + if( cmdRun.withTrace != '-' ) + config.trace.file = cmdRun.withTrace + else if( !config.trace.file ) + config.trace.file = TraceFileObserver.DEF_FILE_NAME + } + + // -- sets report report options + if( cmdRun.withReport ) { + if( !(config.report instanceof Map) ) + config.report = [:] + config.report.enabled = true + if( cmdRun.withReport != '-' ) + config.report.file = cmdRun.withReport + else if( !config.report.file ) + config.report.file = ReportObserver.DEF_FILE_NAME + } + + // -- sets timeline report options + if( cmdRun.withTimeline ) { + if( !(config.timeline instanceof Map) ) + config.timeline = [:] + config.timeline.enabled = true + if( cmdRun.withTimeline != '-' ) + config.timeline.file = cmdRun.withTimeline + else if( !config.timeline.file ) + config.timeline.file = TimelineObserver.DEF_FILE_NAME + } + + // -- sets DAG report options + if( cmdRun.withDag ) { + if( !(config.dag instanceof Map) ) + config.dag = [:] + config.dag.enabled = true + if( cmdRun.withDag != '-' ) + config.dag.file = cmdRun.withDag + else if( !config.dag.file ) + config.dag.file = GraphObserver.DEF_FILE_NAME + } + + if( cmdRun.withNotification ) { + if( !(config.notification instanceof Map) ) + config.notification = [:] + if( cmdRun.withNotification in ['true','false']) { + config.notification.enabled = cmdRun.withNotification == 'true' + } + else { + config.notification.enabled = true + config.notification.to = cmdRun.withNotification + } + } + + // -- sets the messages options + if( cmdRun.withWebLog ) { + log.warn "The command line option '-with-weblog' is deprecated - consider enabling this feature by setting 'weblog.enabled=true' in your configuration file" + if( !(config.weblog instanceof Map) ) + config.weblog = [:] + config.weblog.enabled = true + if( cmdRun.withWebLog != '-' ) + config.weblog.url = cmdRun.withWebLog + else if( !config.weblog.url ) + config.weblog.url = 'http://localhost' + } + + // -- sets tower options + if( cmdRun.withTower ) { + if( !(config.tower instanceof Map) ) + config.tower = [:] + config.tower.enabled = true + if( cmdRun.withTower != '-' ) + config.tower.endpoint = cmdRun.withTower + else if( !config.tower.endpoint ) + config.tower.endpoint = 'https://api.cloud.seqera.io' + } + + // -- set wave options + if( cmdRun.withWave ) { + if( !(config.wave instanceof Map) ) + config.wave = [:] + config.wave.enabled = true + if( cmdRun.withWave != '-' ) + config.wave.endpoint = cmdRun.withWave + else if( !config.wave.endpoint ) + config.wave.endpoint = 'https://wave.seqera.io' + } + + // -- set fusion options + if( cmdRun.withFusion ) { + if( !(config.fusion instanceof Map) ) + config.fusion = [:] + config.fusion.enabled = cmdRun.withFusion == 'true' + } + + // -- set cloudcache options + final envCloudPath = env.get('NXF_CLOUDCACHE_PATH') + if( cmdRun.cloudCachePath || envCloudPath ) { + if( !(config.cloudcache instanceof Map) ) + config.cloudcache = [:] + if( !config.cloudcache.isSet('enabled') ) + config.cloudcache.enabled = true + if( cmdRun.cloudCachePath && cmdRun.cloudCachePath != '-' ) + config.cloudcache.path = cmdRun.cloudCachePath + else if( !config.cloudcache.isSet('path') && envCloudPath ) + config.cloudcache.path = envCloudPath + } + + // -- add the command line parameters to the config + if( cmdRun.hasParams() ) + config.params = mergeMaps( (Map)config.params, cmdRun.parsedParams(builder.configVars()), NF.strictMode ) + + if( cmdRun.withoutDocker && config.docker instanceof Map ) { + // disable docker execution + log.debug "Disabling execution in Docker container as requested by command-line option `-without-docker`" + config.docker.enabled = false + } + + if( cmdRun.withDocker ) { + configContainer(config, 'docker', cmdRun.withDocker) + } + + if( cmdRun.withPodman ) { + configContainer(config, 'podman', cmdRun.withPodman) + } + + if( cmdRun.withSingularity ) { + configContainer(config, 'singularity', cmdRun.withSingularity) + } + + if( cmdRun.withApptainer ) { + configContainer(config, 'apptainer', cmdRun.withApptainer) + } + + if( cmdRun.withCharliecloud ) { + configContainer(config, 'charliecloud', cmdRun.withCharliecloud) + } + } + + private String normalizeResumeId( String uniqueId ) { + if( !uniqueId ) + return null + if( uniqueId == 'last' || uniqueId == 'true' ) { + if( HistoryFile.disabled() ) + throw new AbortOperationException("The resume session id should be specified via `-resume` option when history file tracking is disabled") + uniqueId = HistoryFile.DEFAULT.getLast()?.sessionId + + if( !uniqueId ) { + log.warn "It appears you have never run this project before -- Option `-resume` is ignored" + } + } + + return uniqueId + } + + @CompileDynamic + private void configContainer(ConfigObject config, String engine, def cli) { + log.debug "Enabling execution in ${engine.capitalize()} container as requested by command-line option `-with-$engine ${cmdRun.withDocker}`" + + if( !config.containsKey(engine) ) + config.put(engine, [:]) + + if( !(config.get(engine) instanceof Map) ) + throw new AbortOperationException("Invalid `$engine` definition in the config file") + + def containerConfig = (Map)config.get(engine) + containerConfig.enabled = true + if( cli != '-' ) { + // this is supposed to be a docker image name + config.process.container = cli + } + else if( containerConfig.image ) { + config.process.container = containerConfig.image + } + + if( !hasContainerDirective(config.process) ) + throw new AbortOperationException("You have requested to run with ${engine.capitalize()} but no image was specified") + + } + + /** + * Verify that configuration for process contains at last one `container` directive + * + * @param process + * @return {@code true} when a `container` is defined or {@code false} otherwise + */ + @CompileDynamic + protected boolean hasContainerDirective(process) { + + if( process instanceof Map ) { + if( process.container ) + return true + + def result = process + .findAll { String name, value -> (name.startsWith('withName:') || name.startsWith('$')) && value instanceof Map } + .find { String name, Map value -> value.container as boolean } // the first non-empty `container` string + + return result as boolean + } + + return false + } + + /** + * Merge two maps recursively avoiding keys to be overwritten + * + * @param config + * @param params + * @return a map resulting of merging result and right maps + */ + protected Map mergeMaps(Map config, Map params, boolean strict, List keys=[]) { + if( config==null ) + config = new LinkedHashMap() + + for( Map.Entry entry : params ) { + final key = entry.key.toString() + final value = entry.value + final previous = getConfigVal0(config, key) + keys << entry.key + + if( previous==null ) { + config[key] = value + } + else if( previous instanceof Map && value instanceof Map ) { + mergeMaps(previous, value, strict, keys) + } + else { + if( previous instanceof Map || value instanceof Map ) { + final msg = "Configuration setting type with key '${keys.join('.')}' does not match the parameter with the same key - Config value=$previous; parameter value=$value" + if(strict) + throw new AbortOperationException(msg) + log.warn(msg) + } + config[key] = value + } + } + + return config + } + + private Object getConfigVal0(Map config, String key) { + if( config instanceof ConfigObject ) { + return config.isSet(key) ? config.get(key) : null + } + else { + return config.get(key) + } + } + + static String resolveConfig(Path baseDir, CmdRun cmdRun) { + + final builder = new ConfigBuilder() + .setShowClosures(true) + .setStripSecrets(true) + + final config = new ConfigCmdAdapter(builder) + .setOptions(cmdRun.launcher.options) + .setCmdRun(cmdRun) + .setBaseDir(baseDir) + .buildConfigObject() + + // strip secret + SecretHelper.hideSecrets(config) + // compute config + final result = toCanonicalString(config, false) + // dump config for debugging + log.trace "Resolved config:\n${result.indent('\t')}" + return result + } + + protected static ConfigMap toConfigMap(ConfigObject config) { + assert config != null + (ConfigMap)normalize0((Map)config) + } + + private static Object normalize0( config ) { + + if( config instanceof Map ) { + ConfigMap result = new ConfigMap(config.size()) + for( String name : config.keySet() ) { + def value = (config as Map).get(name) + result.put(name, normalize0(value)) + } + return result + } + else if( config instanceof Collection ) { + List result = new ArrayList(config.size()) + for( entry in config ) { + result << normalize0(entry) + } + return result + } + else { + return config + } + } +} diff --git a/modules/nextflow/src/test/groovy/nextflow/cli/CmdCleanTest.groovy b/modules/nf-cli-v1/src/test/groovy/nextflow/cli/CmdCleanTest.groovy similarity index 100% rename from modules/nextflow/src/test/groovy/nextflow/cli/CmdCleanTest.groovy rename to modules/nf-cli-v1/src/test/groovy/nextflow/cli/CmdCleanTest.groovy diff --git a/modules/nextflow/src/test/groovy/nextflow/cli/CmdCloneTest.groovy b/modules/nf-cli-v1/src/test/groovy/nextflow/cli/CmdCloneTest.groovy similarity index 100% rename from modules/nextflow/src/test/groovy/nextflow/cli/CmdCloneTest.groovy rename to modules/nf-cli-v1/src/test/groovy/nextflow/cli/CmdCloneTest.groovy diff --git a/modules/nextflow/src/test/groovy/nextflow/cli/CmdConfigTest.groovy b/modules/nf-cli-v1/src/test/groovy/nextflow/cli/CmdConfigTest.groovy similarity index 100% rename from modules/nextflow/src/test/groovy/nextflow/cli/CmdConfigTest.groovy rename to modules/nf-cli-v1/src/test/groovy/nextflow/cli/CmdConfigTest.groovy diff --git a/modules/nextflow/src/test/groovy/nextflow/cli/CmdHelperTest.groovy b/modules/nf-cli-v1/src/test/groovy/nextflow/cli/CmdHelperTest.groovy similarity index 100% rename from modules/nextflow/src/test/groovy/nextflow/cli/CmdHelperTest.groovy rename to modules/nf-cli-v1/src/test/groovy/nextflow/cli/CmdHelperTest.groovy diff --git a/modules/nextflow/src/test/groovy/nextflow/cli/CmdInfoTest.groovy b/modules/nf-cli-v1/src/test/groovy/nextflow/cli/CmdInfoTest.groovy similarity index 100% rename from modules/nextflow/src/test/groovy/nextflow/cli/CmdInfoTest.groovy rename to modules/nf-cli-v1/src/test/groovy/nextflow/cli/CmdInfoTest.groovy diff --git a/modules/nextflow/src/test/groovy/nextflow/cli/CmdInspectTest.groovy b/modules/nf-cli-v1/src/test/groovy/nextflow/cli/CmdInspectTest.groovy similarity index 100% rename from modules/nextflow/src/test/groovy/nextflow/cli/CmdInspectTest.groovy rename to modules/nf-cli-v1/src/test/groovy/nextflow/cli/CmdInspectTest.groovy diff --git a/modules/nextflow/src/test/groovy/nextflow/cli/CmdLineageTest.groovy b/modules/nf-cli-v1/src/test/groovy/nextflow/cli/CmdLineageTest.groovy similarity index 100% rename from modules/nextflow/src/test/groovy/nextflow/cli/CmdLineageTest.groovy rename to modules/nf-cli-v1/src/test/groovy/nextflow/cli/CmdLineageTest.groovy diff --git a/modules/nextflow/src/test/groovy/nextflow/cli/CmdLogTest.groovy b/modules/nf-cli-v1/src/test/groovy/nextflow/cli/CmdLogTest.groovy similarity index 100% rename from modules/nextflow/src/test/groovy/nextflow/cli/CmdLogTest.groovy rename to modules/nf-cli-v1/src/test/groovy/nextflow/cli/CmdLogTest.groovy diff --git a/modules/nextflow/src/test/groovy/nextflow/cli/CmdPluginCreateTest.groovy b/modules/nf-cli-v1/src/test/groovy/nextflow/cli/CmdPluginCreateTest.groovy similarity index 100% rename from modules/nextflow/src/test/groovy/nextflow/cli/CmdPluginCreateTest.groovy rename to modules/nf-cli-v1/src/test/groovy/nextflow/cli/CmdPluginCreateTest.groovy diff --git a/modules/nextflow/src/test/groovy/nextflow/cli/CmdPullTest.groovy b/modules/nf-cli-v1/src/test/groovy/nextflow/cli/CmdPullTest.groovy similarity index 100% rename from modules/nextflow/src/test/groovy/nextflow/cli/CmdPullTest.groovy rename to modules/nf-cli-v1/src/test/groovy/nextflow/cli/CmdPullTest.groovy diff --git a/modules/nextflow/src/test/groovy/nextflow/cli/CmdRunTest.groovy b/modules/nf-cli-v1/src/test/groovy/nextflow/cli/CmdRunTest.groovy similarity index 100% rename from modules/nextflow/src/test/groovy/nextflow/cli/CmdRunTest.groovy rename to modules/nf-cli-v1/src/test/groovy/nextflow/cli/CmdRunTest.groovy diff --git a/modules/nextflow/src/test/groovy/nextflow/cli/CmdSecretTest.groovy b/modules/nf-cli-v1/src/test/groovy/nextflow/cli/CmdSecretTest.groovy similarity index 100% rename from modules/nextflow/src/test/groovy/nextflow/cli/CmdSecretTest.groovy rename to modules/nf-cli-v1/src/test/groovy/nextflow/cli/CmdSecretTest.groovy diff --git a/modules/nextflow/src/test/groovy/nextflow/scm/HubOptionsTest.groovy b/modules/nf-cli-v1/src/test/groovy/nextflow/cli/HubAwareTest.groovy similarity index 82% rename from modules/nextflow/src/test/groovy/nextflow/scm/HubOptionsTest.groovy rename to modules/nf-cli-v1/src/test/groovy/nextflow/cli/HubAwareTest.groovy index 0d9e38068c..db7c770cfc 100644 --- a/modules/nextflow/src/test/groovy/nextflow/scm/HubOptionsTest.groovy +++ b/modules/nf-cli-v1/src/test/groovy/nextflow/cli/HubAwareTest.groovy @@ -14,33 +14,29 @@ * limitations under the License. */ -package nextflow.scm +package nextflow.cli -import nextflow.cli.HubOptions import spock.lang.Specification /** * * @author Paolo Di Tommaso */ -class HubOptionsTest extends Specification { +class HubAwareTest extends Specification { def testUser() { when: - def cmd = [:] as HubOptions + def cmd = [:] as HubAware cmd.hubUser = credential then: - cmd.getHubUser() == user - cmd.getHubPassword() == password + cmd.toHubOptions().getUser() == user + cmd.toHubOptions().getPassword() == password where: credential | user | password null | null | null 'paolo' | 'paolo' | null 'paolo:secret' | 'paolo' | 'secret' - - - } } diff --git a/modules/nextflow/src/test/groovy/nextflow/cli/LauncherTest.groovy b/modules/nf-cli-v1/src/test/groovy/nextflow/cli/LauncherTest.groovy similarity index 98% rename from modules/nextflow/src/test/groovy/nextflow/cli/LauncherTest.groovy rename to modules/nf-cli-v1/src/test/groovy/nextflow/cli/LauncherTest.groovy index 2527d1bac1..340aacb4d4 100644 --- a/modules/nextflow/src/test/groovy/nextflow/cli/LauncherTest.groovy +++ b/modules/nf-cli-v1/src/test/groovy/nextflow/cli/LauncherTest.groovy @@ -101,8 +101,8 @@ class LauncherTest extends Specification { launcher.command instanceof CmdPull launcher.command.args == ['xxx'] launcher.command.hubProvider == 'bitbucket' - launcher.command.hubUser == 'xx' - launcher.command.hubPassword == '11' + launcher.command.toHubOptions().getUser() == 'xx' + launcher.command.toHubOptions().getPassword() == '11' } @@ -113,8 +113,8 @@ class LauncherTest extends Specification { launcher.command instanceof CmdClone launcher.command.args == ['xxx'] launcher.command.hubProvider == 'bitbucket' - launcher.command.hubUser == 'xx' - launcher.command.hubPassword == 'yy' + launcher.command.toHubOptions().getUser() == 'xx' + launcher.command.toHubOptions().getPassword() == 'yy' } @@ -125,8 +125,8 @@ class LauncherTest extends Specification { launcher.command instanceof CmdRun launcher.command.args == ['xxx'] launcher.command.hubProvider == 'bitbucket' - launcher.command.hubUser == 'xx' - launcher.command.hubPassword == 'yy' + launcher.command.toHubOptions().getUser() == 'xx' + launcher.command.toHubOptions().getPassword() == 'yy' when: launcher = new Launcher().parseMainArgs('run','alpha', '-hub', 'github') diff --git a/modules/nf-cli-v1/src/test/groovy/nextflow/config/ConfigCmdAdapterTest.groovy b/modules/nf-cli-v1/src/test/groovy/nextflow/config/ConfigCmdAdapterTest.groovy new file mode 100644 index 0000000000..a75048dc40 --- /dev/null +++ b/modules/nf-cli-v1/src/test/groovy/nextflow/config/ConfigCmdAdapterTest.groovy @@ -0,0 +1,1842 @@ +/* + * Copyright 2013-2024, Seqera Labs + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package nextflow.config + +import java.nio.file.Files +import java.nio.file.Path + +import nextflow.cli.CliOptions +import nextflow.cli.CmdConfig +import nextflow.cli.CmdNode +import nextflow.cli.CmdRun +import nextflow.cli.Launcher +import nextflow.exception.AbortOperationException +import nextflow.trace.TraceHelper +import spock.lang.Specification +import spock.lang.Unroll +/** + * + * @author Paolo Di Tommaso + */ +class ConfigCmdAdapterTest extends Specification { + + def setup() { + TraceHelper.testTimestampFmt = '20221001' + } + + def 'CLI params should override the ones defined in the config file' () { + setup: + def file = Files.createTempFile('test',null) + file.text = ''' + params { + alpha = 'x' + } + params.beta = 'y' + params.delta = 'Foo' + params.gamma = params.alpha + params { + omega = 'Bar' + } + + process { + publishDir = [path: params.alpha] + } + ''' + when: + def opt = new CliOptions() + def run = new CmdRun(params: [alpha: 'Hello', beta: 'World', omega: 'Last']) + def result = new ConfigCmdAdapter().setOptions(opt).setCmdRun(run).buildGivenFiles(file) + + then: + result.params.alpha == 'Hello' // <-- params defined as CLI options override the ones in the config file + result.params.beta == 'World' // <-- as above + result.params.gamma == 'Hello' // <-- as above + result.params.omega == 'Last' + result.params.delta == 'Foo' + result.process.publishDir == [path: 'Hello'] + + cleanup: + file?.delete() + } + + def 'CLI params should override the ones defined in the config file [2]' () { + setup: + def file = Files.createTempFile('test',null) + file.text = ''' + params { + alpha = 'x' + beta = 'y' + delta = 'Foo' + gamma = params.alpha + omega = 'Bar' + } + + process { + publishDir = [path: params.alpha] + } + ''' + when: + def opt = new CliOptions() + def run = new CmdRun(params: [alpha: 'Hello', beta: 'World', omega: 'Last']) + def result = new ConfigCmdAdapter().setOptions(opt).setCmdRun(run).buildGivenFiles(file) + + then: + result.params.alpha == 'Hello' // <-- params defined as CLI options override the ones in the config file + result.params.beta == 'World' // <-- as above + result.params.gamma == 'Hello' // <-- as above + result.params.omega == 'Last' + result.params.delta == 'Foo' + result.process.publishDir == [path: 'Hello'] + + cleanup: + file?.delete() + } + + def 'CLI params should override the ones in one or more config files' () { + given: + def folder = File.createTempDir() + def configMain = new File(folder,'nextflow.config').absoluteFile + def snippet1 = new File(folder,'config1.txt').absoluteFile + def snippet2 = new File(folder,'config2.txt').absoluteFile + + + configMain.text = """ + process.name = 'alpha' + params.one = 'a' + params.xxx = 'x' + includeConfig "$snippet1" + """ + + snippet1.text = """ + params.two = 'b' + params.yyy = 'y' + + process.cpus = 4 + process.memory = '8GB' + + includeConfig("$snippet2") + """ + + snippet2.text = ''' + params.three = 'c' + params.zzz = 'z' + + process { disk = '1TB' } + process.resources.foo = 1 + process.resources.bar = 2 + ''' + + when: + def opt = new CliOptions() + def run = new CmdRun(params: [one: '1', two: 'dos', three: 'tres']) + def config = new ConfigCmdAdapter().setOptions(opt).setCmdRun(run).buildGivenFiles(configMain.toPath()) + + then: + config.params.one == 1 + config.params.two == 'dos' + config.params.three == 'tres' + config.process.name == 'alpha' + config.params.xxx == 'x' + config.params.yyy == 'y' + config.params.zzz == 'z' + + config.process.cpus == 4 + config.process.memory == '8GB' + config.process.disk == '1TB' + config.process.resources.foo == 1 + config.process.resources.bar == 2 + + cleanup: + folder?.deleteDir() + } + + def 'should include config with params' () { + given: + def folder = File.createTempDir() + def configMain = new File(folder,'nextflow.config').absoluteFile + def snippet1 = new File(folder,'igenomes.config').absoluteFile + + + configMain.text = ''' + includeConfig 'igenomes.config' + ''' + + snippet1.text = ''' + params { + genomes { + 'GRCh37' { + fasta = "${params.igenomes_base}/genome.fa" + bwa = "${params.igenomes_base}/BWAIndex/genome.fa" + } + } + } + ''' + + when: + def opt = new CliOptions() + def run = new CmdRun(params: [igenomes_base: 'test']) + def config = new ConfigCmdAdapter().setOptions(opt).setCmdRun(run).buildGivenFiles(configMain.toPath()) + + then: + config.params.genomes.GRCh37 == [fasta:'test/genome.fa', bwa:'test/BWAIndex/genome.fa'] + + cleanup: + folder?.deleteDir() + } + + def 'should fetch the config path from env var' () { + given: + def folder = File.createTempDir() + def configMain = new File(folder,'my.config').absoluteFile + + + configMain.text = """ + process.name = 'alpha' + params.one = 'a' + params.two = 'b' + """ + + // relative path to current dir + when: + def config = new ConfigCmdAdapter(sysEnv: [NXF_CONFIG_FILE: 'my.config']) .setCurrentDir(folder.toPath()) .build() + then: + config.params.one == 'a' + config.params.two == 'b' + config.process.name == 'alpha' + + // absolute path + when: + config = new ConfigCmdAdapter(sysEnv: [NXF_CONFIG_FILE: configMain.toString()]) .build() + then: + config.params.one == 'a' + config.params.two == 'b' + config.process.name == 'alpha' + + // default should not find it + when: + config = new ConfigCmdAdapter() .build() + then: + config.params == [:] + + cleanup: + folder?.deleteDir() + } + + def 'CLI params should overrides the ones in one or more profiles' () { + + setup: + def file = Files.createTempFile('test',null) + file.text = ''' + params.alpha = 'a' + params.beta = 'b' + params.delta = 'Foo' + params.gamma = params.alpha + + params { + genomes { + 'GRCh37' { + bed12 = '/data/genes.bed' + bismark = '/data/BismarkIndex' + bowtie = '/data/genome' + } + } + } + + profiles { + first { + params.alpha = 'Alpha' + params.omega = 'Omega' + params.gamma = 'First' + process.name = 'Bar' + } + + second { + params.alpha = 'xxx' + params.gamma = 'Second' + process { + publishDir = [path: params.alpha] + } + } + + } + + ''' + + when: + def opt = new CliOptions() + def run = new CmdRun(params: [alpha: 'AAA', beta: 'BBB']) + def config = new ConfigCmdAdapter().setOptions(opt).setCmdRun(run).buildGivenFiles(file) + then: + config.params.alpha == 'AAA' + config.params.beta == 'BBB' + config.params.delta == 'Foo' + config.params.gamma == 'AAA' + config.params.genomes.GRCh37.bed12 == '/data/genes.bed' + config.params.genomes.GRCh37.bismark == '/data/BismarkIndex' + config.params.genomes.GRCh37.bowtie == '/data/genome' + + when: + opt = new CliOptions() + run = new CmdRun(params: [alpha: 'AAA', beta: 'BBB'], profile: 'first') + config = new ConfigCmdAdapter().setOptions(opt).setCmdRun(run).buildGivenFiles(file) + then: + config.params.alpha == 'AAA' + config.params.beta == 'BBB' + config.params.delta == 'Foo' + config.params.gamma == 'First' + config.process.name == 'Bar' + config.params.genomes.GRCh37.bed12 == '/data/genes.bed' + config.params.genomes.GRCh37.bismark == '/data/BismarkIndex' + config.params.genomes.GRCh37.bowtie == '/data/genome' + + when: + opt = new CliOptions() + run = new CmdRun(params: [alpha: 'AAA', beta: 'BBB', genomes: 'xxx'], profile: 'second') + config = new ConfigCmdAdapter().setOptions(opt).setCmdRun(run).buildGivenFiles(file) + then: + config.params.alpha == 'AAA' + config.params.beta == 'BBB' + config.params.delta == 'Foo' + config.params.gamma == 'Second' + config.params.genomes == 'xxx' + config.process.publishDir == [path: 'AAA'] + + cleanup: + file?.delete() + } + + def 'params-file should override params in the config file' () { + setup: + def baseDir = Path.of('/my/base/dir') + and: + def params = Files.createTempFile('test', '.yml') + params.text = ''' + alpha: "Hello" + beta: "World" + omega: "Last" + theta: "${baseDir}/something" + '''.stripIndent() + and: + def file = Files.createTempFile('test',null) + file.text = ''' + params { + alpha = 'x' + } + params.beta = 'y' + params.delta = 'Foo' + params.gamma = params.alpha + params { + omega = 'Bar' + } + + process { + publishDir = [path: params.alpha] + } + ''' + when: + def opt = new CliOptions() + def run = new CmdRun(paramsFile: params) + def result = new ConfigCmdAdapter().setOptions(opt).setCmdRun(run).setBaseDir(baseDir).buildGivenFiles(file) + + then: + result.params.alpha == 'Hello' // <-- params defined in the params-file overrides the ones in the config file + result.params.beta == 'World' // <-- as above + result.params.gamma == 'Hello' // <-- as above + result.params.omega == 'Last' + result.params.delta == 'Foo' + result.params.theta == "$baseDir/something" + result.process.publishDir == [path: 'Hello'] + + cleanup: + file?.delete() + params?.delete() + } + + def 'params should override params-file and override params in the config file' () { + setup: + def params = Files.createTempFile('test', '.yml') + params.text = ''' + alpha: "Hello" + beta: "World" + omega: "Last" + '''.stripIndent() + and: + def file = Files.createTempFile('test',null) + file.text = ''' + params { + alpha = 'x' + } + params.beta = 'y' + params.delta = 'Foo' + params.gamma = "I'm gamma" + params.omega = "I'm the last" + + process { + publishDir = [path: params.alpha] + } + ''' + when: + def opt = new CliOptions() + def run = new CmdRun(paramsFile: params, params: [alpha: 'Hola', beta: 'Mundo']) + def result = new ConfigCmdAdapter().setOptions(opt).setCmdRun(run).buildGivenFiles(file) + + then: + result.params.alpha == 'Hola' // <-- this comes from the CLI + result.params.beta == 'Mundo' // <-- this comes from the CLI as well + result.params.omega == 'Last' // <-- this comes from the params-file + result.params.gamma == "I'm gamma" // <-- from the config + result.params.delta == 'Foo' // <-- from the config + result.process.publishDir == [path: 'Hola'] + + cleanup: + file?.delete() + params?.delete() + } + + def 'should validate config files' () { + + given: + def folder = Files.createTempDirectory('test') + def f1 = folder.resolve('file1') + def f2 = folder.resolve('file2') + + when: + new ConfigCmdAdapter() + .validateConfigFiles([f1, f2]) + then: + thrown(AbortOperationException) + + when: + f1.text = '1'; f2.text = '2' + def files = new ConfigCmdAdapter() + .resolveConfigFiles([f1.toString(), f2.toString()]) + then: + files == [f1, f2] + + when: + files = new ConfigCmdAdapter() + .setHomeDir(folder) + .setCurrentDir(folder) + .setUserConfigFiles(f1,f2) + .resolveConfigFiles([]) + then: + files == [f1, f2] + + cleanup: + folder.deleteDir() + + } + + def 'should discover default config files' () { + given: + def homeDir = Files.createTempDirectory('home') + def baseDir = Files.createTempDirectory('work') + def workDir = Files.createTempDirectory('work') + + when: + def homeConfig = homeDir.resolve('config') + homeConfig.text = 'foo=1' + def files1 = new ConfigCmdAdapter(homeDir: homeDir, baseDir: workDir, currentDir: workDir).resolveConfigFiles() + then: + files1 == [homeConfig] + + when: + def workConfig = workDir.resolve('nextflow.config') + workConfig.text = 'bar=2' + def files2 = new ConfigCmdAdapter(homeDir: homeDir, baseDir: workDir, currentDir: workDir).resolveConfigFiles() + then: + files2 == [homeConfig, workConfig] + + when: + def baseConfig = baseDir.resolve('nextflow.config') + baseConfig.text = 'ciao=3' + def files3 = new ConfigCmdAdapter(homeDir: homeDir, baseDir: baseDir, currentDir: workDir).resolveConfigFiles() + then: + files3 == [homeConfig, baseConfig, workConfig] + + cleanup: + homeDir?.deleteDir() + workDir?.deleteDir() + } + + def 'command executor options'() { + + when: + def opt = new CliOptions() + def run = new CmdRun(executorOptions: [ alpha: 1, 'beta.x': 'hola', 'beta.y': 'ciao' ]) + def result = new ConfigCmdAdapter().setOptions(opt).setCmdRun(run).buildGivenFiles() + then: + result.executor.alpha == 1 + result.executor.beta.x == 'hola' + result.executor.beta.y == 'ciao' + + } + + def 'run command cluster options'() { + + when: + def opt = new CliOptions() + def run = new CmdRun(clusterOptions: [ alpha: 1, 'beta.x': 'hola', 'beta.y': 'ciao' ]) + def result = new ConfigCmdAdapter().setOptions(opt).setCmdRun(run).buildGivenFiles() + then: + result.cluster.alpha == 1 + result.cluster.beta.x == 'hola' + result.cluster.beta.y == 'ciao' + + } + + def 'run with docker'() { + + when: + def opt = new CliOptions() + def run = new CmdRun(withDocker: 'cbcrg/piper') + def config = new ConfigCmdAdapter().setOptions(opt).setCmdRun(run).build() + + then: + config.docker.enabled + config.process.container == 'cbcrg/piper' + + } + + def 'run with docker 2'() { + + given: + def file = Files.createTempFile('test','config') + file.deleteOnExit() + file.text = + ''' + docker { + image = 'busybox' + enabled = false + } + ''' + + when: + def opt = new CliOptions(config: [file.toFile().canonicalPath] ) + def run = new CmdRun(withDocker: '-') + def config = new ConfigCmdAdapter().setOptions(opt).setCmdRun(run).build() + then: + config.docker.enabled + config.docker.image == 'busybox' + config.process.container == 'busybox' + + when: + opt = new CliOptions(config: [file.toFile().canonicalPath] ) + run = new CmdRun(withDocker: 'cbcrg/mybox') + config = new ConfigCmdAdapter().setOptions(opt).setCmdRun(run).build() + then: + config.docker.enabled + config.process.container == 'cbcrg/mybox' + + } + + def 'run with docker 3'() { + given: + def file = Files.createTempFile('test','config') + file.deleteOnExit() + + when: + file.text = + ''' + process.'withName:test'.container = 'busybox' + ''' + def opt = new CliOptions(config: [file.toFile().canonicalPath]) + def run = new CmdRun(withDocker: '-') + def config = new ConfigCmdAdapter().setOptions(opt).setCmdRun(run).build() + then: + config.docker.enabled + config.process.'withName:test'.container == 'busybox' + + when: + file.text = + ''' + process.container = 'busybox' + ''' + opt = new CliOptions(config: [file.toFile().canonicalPath]) + run = new CmdRun(withDocker: '-') + config = new ConfigCmdAdapter().setOptions(opt).setCmdRun(run).build() + then: + config.docker.enabled + config.process.container == 'busybox' + + when: + opt = new CliOptions() + run = new CmdRun(withDocker: '-') + new ConfigCmdAdapter().setOptions(opt).setCmdRun(run).build() + then: + def e = thrown(AbortOperationException) + e.message == 'You have requested to run with Docker but no image was specified' + + when: + file.text = + ''' + process.'withName:test'.tag = 'tag' + ''' + opt = new CliOptions(config: [file.toFile().canonicalPath]) + run = new CmdRun(withDocker: '-') + new ConfigCmdAdapter().setOptions(opt).setCmdRun(run).build() + then: + e = thrown(AbortOperationException) + e.message == 'You have requested to run with Docker but no image was specified' + + } + + def 'run without docker'() { + + given: + def file = Files.createTempFile('test','config') + file.deleteOnExit() + file.text = + ''' + docker { + image = 'busybox' + enabled = true + } + ''' + + when: + def opt = new CliOptions(config: [file.toFile().canonicalPath] ) + def run = new CmdRun(withoutDocker: true) + def config = new ConfigCmdAdapter().setOptions(opt).setCmdRun(run).build() + then: + !config.docker.enabled + config.docker.image == 'busybox' + !config.process.container + + } + + def 'config with cluster options'() { + + when: + def opt = new CliOptions() + def cmd = new CmdNode(clusterOptions: [join: 'x', group: 'y', interface: 'abc', slots: 10, 'tcp.alpha':'uno', 'tcp.beta': 'due']) + + def config = new ConfigCmdAdapter() + .setOptions(opt) + .setCmdNode(cmd) + .build() + + then: + config.cluster.join == 'x' + config.cluster.group == 'y' + config.cluster.interface == 'abc' + config.cluster.slots == 10 + config.cluster.tcp.alpha == 'uno' + config.cluster.tcp.beta == 'due' + + } + + def 'has container directive' () { + when: + def config = new ConfigCmdAdapter() + + then: + !config.hasContainerDirective(null) + !config.hasContainerDirective([:]) + !config.hasContainerDirective([foo: true]) + config.hasContainerDirective([container: 'hello/world']) + !config.hasContainerDirective([foo: 1, bar: 2]) + !config.hasContainerDirective([foo: 1, bar: 2, baz: [container: 'user/repo']]) + config.hasContainerDirective([foo: 1, bar: 2, $baz: [container: 'user/repo']]) + config.hasContainerDirective([foo: 1, bar: 2, 'withName:baz': [container: 'user/repo']]) + + } + + def 'should set session trace options' () { + + given: + def env = [:] + def builder = [:] as ConfigCmdAdapter + + when: + def config = new ConfigObject() + builder.configRunOptions(config, env, new CmdRun()) + + then: + config.trace instanceof Map + !config.trace.enabled + !config.trace.file + + when: + config = new ConfigObject() + builder.configRunOptions(config, env, new CmdRun(withTrace: 'some-file')) + then: + config.trace instanceof Map + config.trace.enabled + config.trace.file == 'some-file' + + when: + config = new ConfigObject() + config.trace.file = 'foo.txt' + builder.configRunOptions(config, env, new CmdRun(withTrace: 'bar.txt')) + then: // command line should override the config file + config.trace instanceof Map + config.trace.enabled + config.trace.file == 'bar.txt' + + when: + config = new ConfigObject() + config.trace.file = 'foo.txt' + builder.configRunOptions(config, env, new CmdRun(withTrace: '-')) + then: // command line should override the config file + config.trace instanceof Map + config.trace.enabled + config.trace.file == 'foo.txt' + + when: + config = new ConfigObject() + builder.configRunOptions(config, env, new CmdRun(withTrace: '-')) + then: // command line should override the config file + config.trace instanceof Map + config.trace.enabled + config.trace.file == 'trace-20221001.txt' + } + + def 'should set session report options' () { + + given: + def env = [:] + def builder = [:] as ConfigCmdAdapter + + when: + def config = new ConfigObject() + builder.configRunOptions(config, env, new CmdRun()) + then: + !config.report + + when: + config = new ConfigObject() + config.report.file = 'foo.html' + builder.configRunOptions(config, env, new CmdRun()) + then: + config.report instanceof Map + !config.report.enabled + config.report.file == 'foo.html' + + when: + config = new ConfigObject() + builder.configRunOptions(config, env, new CmdRun(withReport: 'my-report.html')) + then: + config.report instanceof Map + config.report.enabled + config.report.file == 'my-report.html' + + when: + config = new ConfigObject() + config.report.file = 'this-report.html' + builder.configRunOptions(config, env, new CmdRun(withReport: 'my-report.html')) + then: + config.report instanceof Map + config.report.enabled + config.report.file == 'my-report.html' + + when: + config = new ConfigObject() + config.report.file = 'this-report.html' + builder.configRunOptions(config, env, new CmdRun(withReport: '-')) + then: + config.report instanceof Map + config.report.enabled + config.report.file == 'this-report.html' + + when: + config = new ConfigObject() + builder.configRunOptions(config, env, new CmdRun(withReport: '-')) + then: + config.report instanceof Map + config.report.enabled + config.report.file == 'report-20221001.html' + } + + def 'should set session dag options' () { + + given: + def env = [:] + def builder = [:] as ConfigCmdAdapter + + when: + def config = new ConfigObject() + builder.configRunOptions(config, env, new CmdRun()) + then: + !config.dag + + when: + config = new ConfigObject() + config.dag.file = 'foo-dag.html' + builder.configRunOptions(config, env, new CmdRun()) + then: + config.dag instanceof Map + !config.dag.enabled + config.dag.file == 'foo-dag.html' + + when: + config = new ConfigObject() + builder.configRunOptions(config, env, new CmdRun(withDag: 'my-dag.html')) + then: + config.dag instanceof Map + config.dag.enabled + config.dag.file == 'my-dag.html' + + when: + config = new ConfigObject() + config.dag.file = 'this-dag.html' + builder.configRunOptions(config, env, new CmdRun(withDag: 'my-dag.html')) + then: + config.dag instanceof Map + config.dag.enabled + config.dag.file == 'my-dag.html' + + when: + config = new ConfigObject() + config.dag.file = 'this-dag.html' + builder.configRunOptions(config, env, new CmdRun(withDag: '-')) + then: + config.dag instanceof Map + config.dag.enabled + config.dag.file == 'this-dag.html' + + when: + config = new ConfigObject() + builder.configRunOptions(config, env, new CmdRun(withDag: '-')) + then: + config.dag instanceof Map + config.dag.enabled + config.dag.file == 'dag-20221001.html' + } + + def 'should set session weblog options' () { + + given: + def env = [:] + def builder = [:] as ConfigCmdAdapter + + when: + def config = new ConfigObject() + builder.configRunOptions(config, env, new CmdRun()) + then: + !config.weblog + + when: + config = new ConfigObject() + config.weblog.url = 'http://bar.com' + builder.configRunOptions(config, env, new CmdRun()) + then: + config.weblog instanceof Map + !config.weblog.enabled + config.weblog.url == 'http://bar.com' + + when: + config = new ConfigObject() + builder.configRunOptions(config, env, new CmdRun(withWebLog: 'http://foo.com')) + then: + config.weblog instanceof Map + config.weblog.enabled + config.weblog.url == 'http://foo.com' + + when: + config = new ConfigObject() + config.weblog.enabled = true + config.weblog.url = 'http://bar.com' + builder.configRunOptions(config, env, new CmdRun(withWebLog: 'http://foo.com')) + then: + config.weblog instanceof Map + config.weblog.enabled + config.weblog.url == 'http://foo.com' + + when: + config = new ConfigObject() + config.weblog.enabled = true + config.weblog.url = 'http://bar.com' + builder.configRunOptions(config, env, new CmdRun(withWebLog: '-')) + then: + config.weblog instanceof Map + config.weblog.enabled + config.weblog.url == 'http://bar.com' + + when: + config = new ConfigObject() + config.weblog.enabled = true + builder.configRunOptions(config, env, new CmdRun(withWebLog: '-')) + then: + config.weblog instanceof Map + config.weblog.enabled + config.weblog.url == 'http://localhost' + + } + + def 'should set session timeline options' () { + + given: + def env = [:] + def builder = [:] as ConfigCmdAdapter + + when: + def config = new ConfigObject() + builder.configRunOptions(config, env, new CmdRun()) + then: + !config.timeline + + when: + config = new ConfigObject() + config.timeline.file = 'my-file.html' + builder.configRunOptions(config, env, new CmdRun()) + then: + config.timeline instanceof Map + !config.timeline.enabled + config.timeline.file == 'my-file.html' + + when: + config = new ConfigObject() + builder.configRunOptions(config, env, new CmdRun(withTimeline: 'my-timeline.html')) + then: + config.timeline instanceof Map + config.timeline.enabled + config.timeline.file == 'my-timeline.html' + + when: + config = new ConfigObject() + config.timeline.enabled = true + config.timeline.file = 'this-timeline.html' + builder.configRunOptions(config, env, new CmdRun(withTimeline: 'my-timeline.html')) + then: + config.timeline instanceof Map + config.timeline.enabled + config.timeline.file == 'my-timeline.html' + + when: + config = new ConfigObject() + config.timeline.enabled = true + config.timeline.file = 'my-timeline.html' + builder.configRunOptions(config, env, new CmdRun(withTimeline: '-')) + then: + config.timeline instanceof Map + config.timeline.enabled + config.timeline.file == 'my-timeline.html' + + when: + config = new ConfigObject() + config.timeline.enabled = true + builder.configRunOptions(config, env, new CmdRun(withTimeline: '-')) + then: + config.timeline instanceof Map + config.timeline.enabled + config.timeline.file == 'timeline-20221001.html' + } + + def 'should set tower options' () { + + given: + def env = [:] + def builder = [:] as ConfigCmdAdapter + + when: + def config = new ConfigObject() + builder.configRunOptions(config, env, new CmdRun()) + then: + !config.tower + + when: + config = new ConfigObject() + config.tower.endpoint = 'http://foo.com' + builder.configRunOptions(config, env, new CmdRun()) + then: + config.tower instanceof Map + !config.tower.enabled + config.tower.endpoint == 'http://foo.com' + + when: + config = new ConfigObject() + builder.configRunOptions(config, env, new CmdRun(withTower: 'http://bar.com')) + then: + config.tower instanceof Map + config.tower.enabled + config.tower.endpoint == 'http://bar.com' + + when: + config = new ConfigObject() + config.tower.endpoint = 'http://foo.com' + builder.configRunOptions(config, env, new CmdRun(withTower: '-')) + then: + config.tower instanceof Map + config.tower.enabled + config.tower.endpoint == 'http://foo.com' + + when: + config = new ConfigObject() + builder.configRunOptions(config, env, new CmdRun(withTower: '-')) + then: + config.tower instanceof Map + config.tower.enabled + config.tower.endpoint == 'https://api.cloud.seqera.io' + } + + def 'should set wave options' () { + + given: + def env = [:] + def builder = [:] as ConfigCmdAdapter + + when: + def config = new ConfigObject() + builder.configRunOptions(config, env, new CmdRun()) + then: + !config.wave + + when: + config = new ConfigObject() + config.wave.endpoint = 'http://foo.com' + builder.configRunOptions(config, env, new CmdRun()) + then: + config.wave instanceof Map + !config.wave.enabled + config.wave.endpoint == 'http://foo.com' + + when: + config = new ConfigObject() + builder.configRunOptions(config, env, new CmdRun(withWave: 'http://bar.com')) + then: + config.wave instanceof Map + config.wave.enabled + config.wave.endpoint == 'http://bar.com' + + when: + config = new ConfigObject() + config.wave.endpoint = 'http://foo.com' + builder.configRunOptions(config, env, new CmdRun(withWave: '-')) + then: + config.wave instanceof Map + config.wave.enabled + config.wave.endpoint == 'http://foo.com' + + when: + config = new ConfigObject() + builder.configRunOptions(config, env, new CmdRun(withWave: '-')) + then: + config.wave instanceof Map + config.wave.enabled + config.wave.endpoint == 'https://wave.seqera.io' + } + + def 'should set cloudcache options' () { + + given: + def env = [:] + def builder = [:] as ConfigCmdAdapter + + when: + def config = new ConfigObject() + builder.configRunOptions(config, env, new CmdRun()) + then: + !config.cloudcache + + when: + config = new ConfigObject() + config.cloudcache.path = 's3://foo/bar' + builder.configRunOptions(config, env, new CmdRun()) + then: + config.cloudcache instanceof Map + !config.cloudcache.enabled + config.cloudcache.path == 's3://foo/bar' + + when: + config = new ConfigObject() + builder.configRunOptions(config, env, new CmdRun(cloudCachePath: 's3://this/that')) + then: + config.cloudcache instanceof Map + config.cloudcache.enabled + config.cloudcache.path == 's3://this/that' + + when: + config = new ConfigObject() + builder.configRunOptions(config, env, new CmdRun(cloudCachePath: '-')) + then: + config.cloudcache instanceof Map + config.cloudcache.enabled + !config.cloudcache.path + + when: + config = new ConfigObject() + config.cloudcache.path = 's3://alpha/delta' + builder.configRunOptions(config, env, new CmdRun(cloudCachePath: '-')) + then: + config.cloudcache instanceof Map + config.cloudcache.enabled + config.cloudcache.path == 's3://alpha/delta' + + when: + config = new ConfigObject() + config.cloudcache.path = 's3://alpha/delta' + builder.configRunOptions(config, env, new CmdRun(cloudCachePath: 's3://should/override/config')) + then: + config.cloudcache instanceof Map + config.cloudcache.enabled + config.cloudcache.path == 's3://should/override/config' + + when: + config = new ConfigObject() + config.cloudcache.enabled = false + builder.configRunOptions(config, env, new CmdRun(cloudCachePath: 's3://should/override/config')) + then: + config.cloudcache instanceof Map + !config.cloudcache.enabled + config.cloudcache.path == 's3://should/override/config' + + when: + config = new ConfigObject() + builder.configRunOptions(config, [NXF_CLOUDCACHE_PATH:'s3://foo'], new CmdRun(cloudCachePath: 's3://should/override/env')) + then: + config.cloudcache instanceof Map + config.cloudcache.enabled + config.cloudcache.path == 's3://should/override/env' + + when: + config = new ConfigObject() + config.cloudcache.path = 's3://config/path' + builder.configRunOptions(config, [NXF_CLOUDCACHE_PATH:'s3://foo'], new CmdRun()) + then: + config.cloudcache instanceof Map + config.cloudcache.enabled + config.cloudcache.path == 's3://config/path' + + when: + config = new ConfigObject() + config.cloudcache.path = 's3://config/path' + builder.configRunOptions(config, [NXF_CLOUDCACHE_PATH:'s3://foo'], new CmdRun(cloudCachePath: 's3://should/override/config')) + then: + config.cloudcache instanceof Map + config.cloudcache.enabled + config.cloudcache.path == 's3://should/override/config' + + } + + def 'should enable conda env' () { + + given: + def env = [:] + def builder = [:] as ConfigCmdAdapter + + when: + def config = new ConfigObject() + builder.configRunOptions(config, env, new CmdRun()) + then: + !config.conda + + when: + config = new ConfigObject() + config.conda.createOptions = 'something' + builder.configRunOptions(config, env, new CmdRun()) + then: + config.conda instanceof Map + !config.conda.enabled + config.conda.createOptions == 'something' + + when: + config = new ConfigObject() + builder.configRunOptions(config, env, new CmdRun(withConda: 'my-recipe.yml')) + then: + config.conda instanceof Map + config.conda.enabled + config.process.conda == 'my-recipe.yml' + + when: + config = new ConfigObject() + config.conda.enabled = true + builder.configRunOptions(config, env, new CmdRun(withConda: 'my-recipe.yml')) + then: + config.conda instanceof Map + config.conda.enabled + config.process.conda == 'my-recipe.yml' + + when: + config = new ConfigObject() + config.process.conda = 'my-recipe.yml' + builder.configRunOptions(config, env, new CmdRun(withConda: '-')) + then: + config.conda instanceof Map + config.conda.enabled + config.process.conda == 'my-recipe.yml' + } + + def 'should disable conda env' () { + given: + def file = Files.createTempFile('test','config') + file.deleteOnExit() + file.text = + ''' + conda { + enabled = true + } + ''' + + when: + def opt = new CliOptions(config: [file.toFile().canonicalPath] ) + def run = new CmdRun(withoutConda: true) + def config = new ConfigCmdAdapter().setOptions(opt).setCmdRun(run).build() + then: + !config.conda.enabled + !config.process.conda + } + + def 'should enable spack env' () { + + given: + def env = [:] + def builder = [:] as ConfigCmdAdapter + + when: + def config = new ConfigObject() + builder.configRunOptions(config, env, new CmdRun()) + then: + !config.spack + + when: + config = new ConfigObject() + config.spack.createOptions = 'something' + builder.configRunOptions(config, env, new CmdRun()) + then: + config.spack instanceof Map + !config.spack.enabled + config.spack.createOptions == 'something' + + when: + config = new ConfigObject() + builder.configRunOptions(config, env, new CmdRun(withSpack: 'my-recipe.yaml')) + then: + config.spack instanceof Map + config.spack.enabled + config.process.spack == 'my-recipe.yaml' + + when: + config = new ConfigObject() + config.spack.enabled = true + builder.configRunOptions(config, env, new CmdRun(withSpack: 'my-recipe.yaml')) + then: + config.spack instanceof Map + config.spack.enabled + config.process.spack == 'my-recipe.yaml' + + when: + config = new ConfigObject() + config.process.spack = 'my-recipe.yaml' + builder.configRunOptions(config, env, new CmdRun(withSpack: '-')) + then: + config.spack instanceof Map + config.spack.enabled + config.process.spack == 'my-recipe.yaml' + } + + def 'should disable spack env' () { + given: + def file = Files.createTempFile('test','config') + file.deleteOnExit() + file.text = + ''' + spack { + enabled = true + } + ''' + + when: + def opt = new CliOptions(config: [file.toFile().canonicalPath] ) + def run = new CmdRun(withoutSpack: true) + def config = new ConfigCmdAdapter().setOptions(opt).setCmdRun(run).build() + then: + !config.spack.enabled + !config.process.spack + } + + def 'SHOULD SET `RESUME` OPTION'() { + + given: + def env = [:] + def builder = [:] as ConfigCmdAdapter + + when: + def config = new ConfigObject() + builder.configRunOptions(config, env, new CmdRun()) + then: + !config.isSet('resume') + + when: + config = new ConfigObject() + config.resume ='alpha-beta-delta' + builder.configRunOptions(config, env, new CmdRun()) + then: + config.resume == 'alpha-beta-delta' + + when: + config = new ConfigObject() + config.resume ='alpha-beta-delta' + builder.configRunOptions(config, env, new CmdRun(resume: 'xxx-yyy')) + then: + config.resume == 'xxx-yyy' + + when: + config = new ConfigObject() + config.resume ='this-that' + builder.configRunOptions(config, env, new CmdRun(resume: 'xxx-yyy')) + then: + config.resume == 'xxx-yyy' + } + + def 'should set `workDir`' () { + + given: + def config = new ConfigObject() + def builder = [:] as ConfigCmdAdapter + + when: + builder.configRunOptions(config, [:], new CmdRun()) + then: + config.workDir == 'work' + + when: + config = new ConfigObject() + builder.configRunOptions(config, [NXF_WORK: '/foo/bar'], new CmdRun()) + then: + config.workDir == '/foo/bar' + + when: + config = new ConfigObject() + config.workDir = 'hello/there' + builder.configRunOptions(config, [:], new CmdRun()) + then: + config.workDir == 'hello/there' + + when: + config = new ConfigObject() + config.workDir = 'hello/there' + builder.configRunOptions(config, [:], new CmdRun(workDir: 'my/work/dir')) + then: + config.workDir == 'my/work/dir' + } + + def 'should set `libDir`' () { + given: + def config = new ConfigObject() + def builder = [:] as ConfigCmdAdapter + + when: + builder.configRunOptions(config, [:], new CmdRun()) + then: + !config.isSet('libDir') + + when: + builder.configRunOptions(config, [NXF_LIB:'/foo/bar'], new CmdRun()) + then: + config.libDir == '/foo/bar' + + when: + builder.configRunOptions(config, [:], new CmdRun(libPath: 'my/lib/dir')) + then: + config.libDir == 'my/lib/dir' + } + + def 'should set `cacheable`' () { + given: + def env = [:] + def config + def builder = [:] as ConfigCmdAdapter + + when: + config = new ConfigObject() + builder.configRunOptions(config, env, new CmdRun()) + then: + !config.isSet('cacheable') + + when: + config = new ConfigObject() + builder.configRunOptions(config, env, new CmdRun(cacheable: false)) + then: + config.cacheable == false + + when: + config = new ConfigObject() + builder.configRunOptions(config, env, new CmdRun(cacheable: true)) + then: + config.cacheable == true + } + + def 'should set profile options' () { + def builder + def adapter + + when: + builder = new ConfigBuilder() + adapter = new ConfigCmdAdapter(builder).setCmdRun(new CmdRun(profile: 'foo')) + then: + builder.profile == 'foo' + builder.validateProfile + + when: + builder = new ConfigBuilder() + adapter = new ConfigCmdAdapter(builder).setCmdRun(new CmdRun()) + then: + builder.profile == 'standard' + !builder.validateProfile + + when: + builder = new ConfigBuilder() + adapter = new ConfigCmdAdapter(builder).setCmdRun(new CmdRun(profile: 'standard')) + then: + builder.profile == 'standard' + builder.validateProfile + } + + def 'should set config options' () { + def builder + def adapter + + when: + builder = new ConfigBuilder() + adapter = new ConfigCmdAdapter(builder).setCmdConfig(new CmdConfig()) + then: + !builder.showAllProfiles + + when: + builder = new ConfigBuilder() + adapter = new ConfigCmdAdapter(builder).setCmdConfig(new CmdConfig(showAllProfiles: true)) + then: + builder.showAllProfiles + + when: + builder = new ConfigBuilder() + adapter = new ConfigCmdAdapter(builder).setCmdConfig(new CmdConfig(profile: 'foo')) + then: + builder.profile == 'foo' + builder.validateProfile + + } + + def 'should set params into config object' () { + + given: + def emptyFile = Files.createTempFile('empty','config').toFile() + def EMPTY = [emptyFile.toString()] + + def configFile = Files.createTempFile('test','config').toFile() + configFile.deleteOnExit() + configFile.text = ''' + params.foo = 1 + params.bar = 2 + params.data = '/some/path' + ''' + configFile = configFile.toString() + + def jsonFile = Files.createTempFile('test','.json').toFile() + jsonFile.text = ''' + { + "foo": 10, + "bar": 20 + } + ''' + jsonFile = jsonFile.toString() + + def yamlFile = Files.createTempFile('test','.yaml').toFile() + yamlFile.text = ''' + { + "foo": 100, + "bar": 200 + } + ''' + yamlFile = yamlFile.toString() + + def config + + when: + config = new ConfigCmdAdapter().setOptions(new CliOptions(config: EMPTY)).setCmdRun(new CmdRun()).build() + then: + config.params == [:] + + // get params for the CLI + when: + config = new ConfigCmdAdapter().setOptions(new CliOptions(config: EMPTY)).setCmdRun(new CmdRun(params: [foo:'one', bar:'two'])).build() + then: + config.params == [foo:'one', bar:'two'] + + // get params from config file + when: + config = new ConfigCmdAdapter().setOptions(new CliOptions(config: [configFile])).setCmdRun(new CmdRun()).build() + then: + config.params == [foo:1, bar:2, data: '/some/path'] + + // get params form JSON file + when: + config = new ConfigCmdAdapter().setOptions(new CliOptions(config: EMPTY)).setCmdRun(new CmdRun(paramsFile: jsonFile)).build() + then: + config.params == [foo:10, bar:20] + + // get params from YAML file + when: + config = new ConfigCmdAdapter().setOptions(new CliOptions(config: EMPTY)).setCmdRun(new CmdRun(paramsFile: yamlFile)).build() + then: + config.params == [foo:100, bar:200] + + // cli override config + when: + config = new ConfigCmdAdapter().setOptions(new CliOptions(config: [configFile])).setCmdRun(new CmdRun(params:[foo:'hello', baz:'world'])).build() + then: + config.params == [foo:'hello', bar:2, baz: 'world', data: '/some/path'] + + // CLI override JSON + when: + config = new ConfigCmdAdapter().setOptions(new CliOptions(config: EMPTY)).setCmdRun(new CmdRun(params:[foo:'hello', baz:'world'], paramsFile: jsonFile)).build() + then: + config.params == [foo:'hello', bar:20, baz: 'world'] + + // JSON override config + when: + config = new ConfigCmdAdapter().setOptions(new CliOptions(config: [configFile])).setCmdRun(new CmdRun(paramsFile: jsonFile)).build() + then: + config.params == [foo:10, bar:20, data: '/some/path'] + + // CLI override JSON that override config + when: + config = new ConfigCmdAdapter().setOptions(new CliOptions(config: [configFile])).setCmdRun(new CmdRun(paramsFile: jsonFile, params: [foo:'Ciao'])).build() + then: + config.params == [foo:'Ciao', bar:20, data: '/some/path'] + } + + def 'should run with conda' () { + + when: + def config = new ConfigCmdAdapter().setCmdRun(new CmdRun(withConda: '/some/path/env.yml')).build() + then: + config.process.conda == '/some/path/env.yml' + + } + + def 'should run with spack' () { + + when: + def config = new ConfigCmdAdapter().setCmdRun(new CmdRun(withSpack: '/some/path/env.yaml')).build() + then: + config.process.spack == '/some/path/env.yaml' + + } + + def 'should configure notification' () { + + given: + Map config + + when: + config = new ConfigCmdAdapter().setCmdRun(new CmdRun()).build() + then: + !config.notification + + when: + config = new ConfigCmdAdapter().setCmdRun(new CmdRun(withNotification: true)).build() + then: + config.notification.enabled == true + + when: + config = new ConfigCmdAdapter().setCmdRun(new CmdRun(withNotification: false)).build() + then: + config.notification.enabled == false + config.notification.to == null + + when: + config = new ConfigCmdAdapter().setCmdRun(new CmdRun(withNotification: 'yo@nextflow.com')).build() + then: + config.notification.enabled == true + config.notification.to == 'yo@nextflow.com' + } + + def 'should configure fusion' () { + + given: + Map config + + when: + config = new ConfigCmdAdapter().setCmdRun(new CmdRun()).build() + then: + !config.fusion + + when: + config = new ConfigCmdAdapter().setCmdRun(new CmdRun(withFusion: true)).build() + then: + config.fusion.enabled == true + + when: + config = new ConfigCmdAdapter().setCmdRun(new CmdRun(withFusion: false)).build() + then: + config.fusion == [enabled: false] + + when: + config = new ConfigCmdAdapter().setCmdRun(new CmdRun(withFusion: true)).build() + then: + config.fusion == [enabled: true] + } + + def 'should configure stub run mode' () { + given: + Map config + + when: + config = new ConfigCmdAdapter().setCmdRun(new CmdRun()).build() + then: + !config.stubRun + + when: + config = new ConfigCmdAdapter().setCmdRun(new CmdRun(stubRun: true)).build() + then: + config.stubRun == true + } + + def 'should configure preview mode' () { + given: + Map config + + when: + config = new ConfigCmdAdapter().setCmdRun(new CmdRun()).build() + then: + !config.preview + + when: + config = new ConfigCmdAdapter().setCmdRun(new CmdRun(preview: true)).build() + then: + config.preview == true + } + + def 'CLI params should overwrite only the key provided when nested'() { + given: + def folder = File.createTempDir() + def configMain = new File(folder, 'nextflow.config').absoluteFile + + configMain.text = """ + params { + foo = 'Hello' + bar = "Monde" + baz { + x = "Ciao" + y = "mundo" + z { + alpha = "Hallo" + beta = "World" + } + } + + } + """ + + when: + def opt = new CliOptions() + def run = new CmdRun(params: [bar: "world", 'baz.y': "mondo", 'baz.z.beta': "Welt"]) + def config = new ConfigCmdAdapter(sysEnv: [NXF_CONFIG_FILE: configMain.toString()]).setOptions(opt).setCmdRun(run).build() + + then: + config.params.foo == 'Hello' + config.params.bar == 'world' + config.params.baz.x == 'Ciao' + config.params.baz.y == 'mondo' + //tests recursion + config.params.baz.z.alpha == 'Hallo' + config.params.baz.z.beta == 'Welt' + + cleanup: + folder?.deleteDir() + } + + @Unroll + def 'should merge config params' () { + given: + def builder = new ConfigCmdAdapter() + + expect: + def cfg = new ConfigObject(); if(CONFIG) cfg.putAll(CONFIG) + and: + builder.mergeMaps(cfg, PARAMS, false) == EXPECTED + + where: + CONFIG | PARAMS | EXPECTED + [foo:1] | null | [foo:1] + null | [bar:2] | [bar:2] + [foo:1] | [bar:2] | [foo: 1, bar: 2] + [foo:1] | [bar:null] | [foo: 1, bar: null] + [foo:1] | [foo:null] | [foo: null] + [foo:1, bar:[:]] | [bar: [x:10, y:20]] | [foo: 1, bar: [x:10, y:20]] + [foo:1, bar:[x:1, y:2]] | [bar: [x:10, y:20]] | [foo: 1, bar: [x:10, y:20]] + [foo:1, bar:[x:1, y:2]] | [foo: 2, bar: [x:10, y:20]] | [foo: 2, bar: [x:10, y:20]] + [foo:1, bar:null] | [bar: [x:10, y:20]] | [foo: 1, bar: [x:10, y:20]] + [foo:1, bar:2] | [bar: [x:10, y:20]] | [foo: 1, bar: [x:10, y:20]] + [foo:1, bar:[x:1, y:2]] | [bar: 2] | [foo: 1, bar: 2] + } + + @Unroll + def 'should merge config strict params' () { + given: + def builder = new ConfigCmdAdapter() + + expect: + def cfg = new ConfigObject(); if(CONFIG) cfg.putAll(CONFIG) + and: + builder.mergeMaps(cfg, PARAMS, true) == EXPECTED + + where: + CONFIG | PARAMS | EXPECTED + [:] | [bar:2] | [bar:2] + [foo:1] | null | [foo:1] + null | [bar:2] | [bar:2] + [foo:1] | [bar:2] | [foo: 1, bar: 2] + [foo:1] | [bar:null] | [foo: 1, bar: null] + [foo:1] | [foo:null] | [foo: null] + [foo:1, bar:[:]] | [bar: [x:10, y:20]] | [foo: 1, bar: [x:10, y:20]] + [foo:1, bar:[x:1, y:2]] | [bar: [x:10, y:20]] | [foo: 1, bar: [x:10, y:20]] + [foo:1, bar:[x:1, y:2]] | [foo: 2, bar: [x:10, y:20]] | [foo: 2, bar: [x:10, y:20]] + } + + def 'prevent config side effects' () { + given: + def folder = Files.createTempDirectory('test') + and: + def config = folder.resolve('nf.config') + config.text = '''\ + params.test.foo = "foo_def" + params.test.bar = "bar_def" + '''.stripIndent() + + when: + def cfg1 = new ConfigCmdAdapter() + .setOptions( new CliOptions(userConfig: [config.toString()])) + .build() + then: + cfg1.params.test.foo == "foo_def" + cfg1.params.test.bar == "bar_def" + + + when: + def cfg2 = new ConfigCmdAdapter() + .setOptions( new CliOptions(userConfig: [config.toString()])) + .setCmdRun( new CmdRun(params: ['test.foo': 'CLI_FOO'] )) + .build() + then: + cfg2.params.test.foo == "CLI_FOO" + cfg2.params.test.bar == "bar_def" + + cleanup: + folder?.deleteDir() + } + + def 'parse nested json' () { + given: + def folder = Files.createTempDirectory('test') + and: + def config = folder.resolve('nf.json') + config.text = '''\ + { + "title": "something", + "nested": { + "name": "Mike", + "and": { + "more": "nesting", + "still": { + "another": "layer" + } + } + } + } + '''.stripIndent() + + when: + def cfg1 = new ConfigCmdAdapter().setCmdRun(new CmdRun(paramsFile: config.toString())).build() + + then: + cfg1.params.title == "something" + cfg1.params.nested.name == 'Mike' + cfg1.params.nested.and.more == 'nesting' + cfg1.params.nested.and.still.another == 'layer' + + cleanup: + folder?.deleteDir() + } + + def 'parse nested yaml' () { + given: + def folder = Files.createTempDirectory('test') + and: + def config = folder.resolve('nf.yaml') + config.text = '''\ + title: "something" + nested: + name: "Mike" + and: + more: nesting + still: + another: layer + '''.stripIndent() + + when: + def cfg1 = new ConfigCmdAdapter().setCmdRun(new CmdRun(paramsFile: config.toString())).build() + + then: + cfg1.params.title == "something" + cfg1.params.nested.name == 'Mike' + cfg1.params.nested.and.more == 'nesting' + cfg1.params.nested.and.still.another == 'layer' + + cleanup: + folder?.deleteDir() + } + + def 'should return parsed config' () { + given: + def cmd = new CmdRun(profile: 'first', withTower: 'http://foo.com', launcher: new Launcher()) + def base = Files.createTempDirectory('test') + base.resolve('nextflow.config').text = ''' + profiles { + first { + params { + foo = 'Hello world' + awsKey = 'xyz' + } + process { + executor = { 'local' } + } + } + second { + params.none = 'Blah' + } + } + ''' + when: + def txt = ConfigCmdAdapter.resolveConfig(base, cmd) + then: + txt == '''\ + params { + foo = 'Hello world' + awsKey = '[secret]' + } + + process { + executor = { 'local' } + } + + workDir = 'work' + + tower { + enabled = true + endpoint = 'http://foo.com' + } + '''.stripIndent() + + cleanup: + base?.deleteDir() + } + +} + diff --git a/modules/nf-lineage/build.gradle b/modules/nf-lineage/build.gradle index 4d7405bfc5..f1fbb4b303 100644 --- a/modules/nf-lineage/build.gradle +++ b/modules/nf-lineage/build.gradle @@ -30,7 +30,7 @@ configurations { } dependencies { - api project(':nextflow') + api project(':nf-cli-v1') testImplementation(testFixtures(project(":nextflow"))) testImplementation "org.apache.groovy:groovy:4.0.26" diff --git a/packing.gradle b/packing.gradle index c22846f31a..dd0670da81 100644 --- a/packing.gradle +++ b/packing.gradle @@ -11,7 +11,7 @@ configurations { } dependencies { - api project(':nextflow') + api project(':nf-cli-v1') // include Ivy at runtime in order to have Grape @Grab work correctly defaultCfg "org.apache.ivy:ivy:2.5.2" // default cfg = runtime + httpfs + lineage + amazon + tower client + wave client @@ -91,21 +91,22 @@ protected coordinates( it ) { /* * Compile and pack all packages */ -task packOne( dependsOn: [compile, ":nextflow:shadowJar"]) { +task packOne( dependsOn: [compile, ":nf-cli-v1:shadowJar"]) { doLast { - def source = "modules/nextflow/build/libs/nextflow-${version}-one.jar" - ant.copy(file: source, todir: releaseDir, overwrite: true) - ant.copy(file: source, todir: nextflowDir, overwrite: true) - println "\n+ Nextflow package `ONE` copied to: $releaseDir/nextflow-${version}-one.jar" + def source = "modules/nf-cli-v1/build/libs/nf-cli-v1-${version}-one.jar" + def targetName = "nextflow-${version}-one.jar" + "cp $source $releaseDir/$targetName".execute() + "cp $source $nextflowDir/$targetName".execute() + println "\n+ Nextflow package `ONE` copied to: $releaseDir/$targetName" } } -task packDist( dependsOn: [compile, ":nextflow:shadowJar"]) { +task packDist( dependsOn: [compile, ":nf-cli-v1:shadowJar"]) { doLast { file(releaseDir).mkdirs() // cleanup - def source = file("modules/nextflow/build/libs/nextflow-${version}-one.jar") + def source = file("modules/nf-cli-v1/build/libs/nf-cli-v1-${version}-one.jar") def target = file("$releaseDir/nextflow-${version}-dist"); target.delete() // append the big jar target.withOutputStream { @@ -115,7 +116,7 @@ task packDist( dependsOn: [compile, ":nextflow:shadowJar"]) { // execute permission "chmod +x $target".execute() // done - println "+ Nextflow package `ALL` copied to: $target\n" + println "+ Nextflow package `DIST` copied to: $target\n" } } @@ -123,7 +124,6 @@ task packDist( dependsOn: [compile, ":nextflow:shadowJar"]) { * Compile and pack all packages */ task pack( dependsOn: [packOne, packDist]) { - } task deploy( type: Exec, dependsOn: [clean, compile, pack]) { diff --git a/plugins/nf-console/build.gradle b/plugins/nf-console/build.gradle index 40f5960f78..a55e31ace0 100644 --- a/plugins/nf-console/build.gradle +++ b/plugins/nf-console/build.gradle @@ -33,7 +33,7 @@ configurations { } dependencies { - compileOnly project(':nextflow') + compileOnly project(':nf-cli-v1') compileOnly 'org.slf4j:slf4j-api:2.0.16' compileOnly 'org.pf4j:pf4j:3.12.0' diff --git a/plugins/nf-console/src/main/nextflow/ui/console/Nextflow.groovy b/plugins/nf-console/src/main/nextflow/ui/console/Nextflow.groovy index d153a61a04..f4f4a93270 100644 --- a/plugins/nf-console/src/main/nextflow/ui/console/Nextflow.groovy +++ b/plugins/nf-console/src/main/nextflow/ui/console/Nextflow.groovy @@ -31,7 +31,7 @@ import nextflow.Session import nextflow.cli.CliOptions import nextflow.cli.CmdInfo import nextflow.cli.CmdRun -import nextflow.config.ConfigBuilder +import nextflow.config.ConfigCmdAdapter import nextflow.script.ScriptBinding import nextflow.script.ScriptFile import nextflow.script.parser.v1.ScriptLoaderV1 @@ -82,7 +82,7 @@ class Nextflow extends Console { final base = script ? script.parent : Paths.get('.') // create the config object - return new ConfigBuilder() + return new ConfigCmdAdapter() .setOptions( new CliOptions() ) .setBaseDir(base) .setCmdRun( new CmdRun() ) diff --git a/plugins/nf-k8s/build.gradle b/plugins/nf-k8s/build.gradle index 454f7bdb8c..f9da39c827 100644 --- a/plugins/nf-k8s/build.gradle +++ b/plugins/nf-k8s/build.gradle @@ -33,7 +33,7 @@ configurations { } dependencies { - compileOnly project(':nextflow') + compileOnly project(':nf-cli-v1') compileOnly 'org.slf4j:slf4j-api:2.0.16' compileOnly 'org.pf4j:pf4j:3.12.0' diff --git a/plugins/nf-k8s/src/main/nextflow/k8s/K8sDriverLauncher.groovy b/plugins/nf-k8s/src/main/nextflow/k8s/K8sDriverLauncher.groovy index c56a684cc6..cd09d989b8 100644 --- a/plugins/nf-k8s/src/main/nextflow/k8s/K8sDriverLauncher.groovy +++ b/plugins/nf-k8s/src/main/nextflow/k8s/K8sDriverLauncher.groovy @@ -30,6 +30,7 @@ import groovy.util.logging.Slf4j import nextflow.cli.CmdKubeRun import nextflow.cli.CmdRun import nextflow.config.ConfigBuilder +import nextflow.config.ConfigCmdAdapter import nextflow.exception.AbortOperationException import nextflow.file.FileHelper import nextflow.k8s.client.K8sClient @@ -265,17 +266,20 @@ class K8sDriverLauncher { // -- load local config if available final builder = new ConfigBuilder() .setShowClosures(true) - .setOptions(cmd.launcher.options) .setProfile(cmd.profile) + + final adapter = new ConfigCmdAdapter(builder) + .setOptions(cmd.launcher.options) .setCmdRun(cmd) if( !interactive && !pipelineName.startsWith('/') && !cmd.remoteProfile && !cmd.runRemoteConfig ) { // -- check and parse project remote config final pipelineConfig = new AssetManager(pipelineName, cmd) .getConfigFile() - builder.setUserConfigFiles(pipelineConfig) + if( pipelineConfig ) + adapter.setUserConfigFiles(pipelineConfig) } - return builder.buildConfigObject() + return adapter.buildConfigObject() } protected K8sConfig makeK8sConfig(Map config) { diff --git a/plugins/nf-tower/build.gradle b/plugins/nf-tower/build.gradle index fb032cbcde..e44fbac0c1 100644 --- a/plugins/nf-tower/build.gradle +++ b/plugins/nf-tower/build.gradle @@ -29,7 +29,7 @@ configurations { } dependencies { - compileOnly project(':nextflow') + compileOnly project(':nf-cli-v1') compileOnly 'org.slf4j:slf4j-api:2.0.16' compileOnly 'org.pf4j:pf4j:3.12.0' diff --git a/plugins/nf-wave/build.gradle b/plugins/nf-wave/build.gradle index f59b99250e..13e4453468 100644 --- a/plugins/nf-wave/build.gradle +++ b/plugins/nf-wave/build.gradle @@ -29,7 +29,7 @@ configurations { } dependencies { - compileOnly project(':nextflow') + compileOnly project(':nf-cli-v1') compileOnly 'org.slf4j:slf4j-api:2.0.16' compileOnly 'org.pf4j:pf4j:3.12.0' compileOnly 'io.seqera:lib-trace:0.1.0' diff --git a/settings.gradle b/settings.gradle index 53d56ba13b..43df6cb7a1 100644 --- a/settings.gradle +++ b/settings.gradle @@ -24,6 +24,7 @@ plugins { rootProject.name = 'nextflow-prj' include 'nextflow' +include 'nf-cli-v1' include 'nf-commons' include 'nf-httpfs' include 'nf-lang'