diff --git a/modules/UMCUGenetics/dragen/main.nf b/modules/UMCUGenetics/dragen/main.nf index cdcef2b..efe6692 100644 --- a/modules/UMCUGenetics/dragen/main.nf +++ b/modules/UMCUGenetics/dragen/main.nf @@ -9,15 +9,15 @@ process DRAGEN { container '079623148045.dkr.ecr.eu-central-1.amazonaws.com/cp-prod/f1b7ad6a-11ac-4bc1-b705-b275ff2887ad:latest' input: - // TODO: Move ref_tar unpacking to a separate process to avoid unpacking the reference for every sample? -> added in prs branch tuple val(meta), path(r1_fastq), path(r2_fastq) path fastq_list - path ref_tar + tuple val(meta2), path(ref_dir) path repeat_genotype_specs output: tuple val(meta), path("*"), emit: output tuple val(meta), path('*.csv'), emit: csv + tuple val(meta), path("${prefix}.bam"), path("${prefix}.bam.bai"), emit: bam_bai tuple val(meta), path("${prefix}.wgs_coverage_metrics.csv"), optional: true, emit: wgs_coverage_metrics tuple val(meta), path("${prefix}.cnv_metrics.csv"), optional: true, emit: cnv_metrics tuple val(meta), path("${prefix}.mapping_metrics.csv"), optional: true, emit: mapping_metrics @@ -31,24 +31,18 @@ process DRAGEN { script: prefix = task.ext.prefix ?: "${meta.id}" def args = task.ext.args ?: '' - - if (repeat_genotype_specs) { - args = args + " --repeat-genotype-enable true --repeat-genotype-specs " + repeat_genotype_specs - } - + def repeat_specs = repeat_genotype_specs ? "--repeat-genotype-enable true --repeat-genotype-specs ${repeat_genotype_specs}" : "" """ - mkdir -p /scratch/reference - tar -C /scratch/reference -xf ${ref_tar} - /opt/edico/bin/dragen --partial-reconfig HMM --ignore-version-check true /opt/edico/bin/dragen --lic-instance-id-location /opt/instance-identity \\ - --ref-dir /scratch/reference/DRAGEN/9 \\ + --ref-dir ${ref_dir} \\ --fastq-list ${fastq_list} \\ --fastq-list-sample-id ${meta.id} \\ --output-file-prefix ${prefix} \\ --output-directory ./ \\ --intermediate-results-dir /scratch \\ + ${repeat_specs} \\ ${args} """ @@ -59,6 +53,7 @@ process DRAGEN { echo ${args} touch ${prefix}.bam + touch ${prefix}.bam.bai touch ${prefix}.wgs_coverage_metrics.csv touch ${prefix}.cnv_metrics.csv touch ${prefix}.mapping_metrics.csv diff --git a/modules/UMCUGenetics/dragen/tests/main.nf.test b/modules/UMCUGenetics/dragen/tests/main.nf.test index 5cdc46c..ad8e822 100644 --- a/modules/UMCUGenetics/dragen/tests/main.nf.test +++ b/modules/UMCUGenetics/dragen/tests/main.nf.test @@ -6,8 +6,34 @@ nextflow_process { tag "modules" tag "modules_UMCUGenetics" + tag "subworkflows/prepare_ica_references" tag "dragen" + + setup{ + nfcoreInitialise("${launchDir}/library/") + nfcoreInstall( + "${launchDir}/library/", + [ + "samtools/faidx", + "gatk4/createsequencedictionary" + ] + ) + nfcoreLink("${launchDir}/library/", "${baseDir}/modules/") + + run("PREPARE_ICA_REFERENCES") { + script "../../../../subworkflows/UMCUGenetics/prepare_ica_references/main.nf" + workflow { + """ + input[0] = channel.of([ + [id: 'genome'], + file(params.subworkflows_umcu_testdata_base_path + "prepare_ica_references/genome.tar.gz", checkIfExists: true) + ]) + """ + } + } + } + test("dragen - stub") { options '-stub' when { @@ -19,7 +45,7 @@ nextflow_process { [] ] input[1] = [] - input[2] = [] + input[2] = PREPARE_ICA_REFERENCES.out.genome_dir input[3] = [] """ } @@ -33,5 +59,8 @@ nextflow_process { ).match() } ) } + cleanup { + nfcoreUnlink("${launchDir}/library/", "${baseDir}/modules/") + } } } diff --git a/modules/UMCUGenetics/dragen/tests/main.nf.test.snap b/modules/UMCUGenetics/dragen/tests/main.nf.test.snap index f18f987..da6efd8 100644 --- a/modules/UMCUGenetics/dragen/tests/main.nf.test.snap +++ b/modules/UMCUGenetics/dragen/tests/main.nf.test.snap @@ -9,6 +9,7 @@ }, [ "sample1.bam:md5,d41d8cd98f00b204e9800998ecf8427e", + "sample1.bam.bai:md5,d41d8cd98f00b204e9800998ecf8427e", "sample1.cnv_metrics.csv:md5,d41d8cd98f00b204e9800998ecf8427e", "sample1.gvcf_metrics.csv:md5,d41d8cd98f00b204e9800998ecf8427e", "sample1.mapping_metrics.csv:md5,d41d8cd98f00b204e9800998ecf8427e", @@ -36,7 +37,8 @@ { "id": "sample1" }, - "sample1.wgs_coverage_metrics.csv:md5,d41d8cd98f00b204e9800998ecf8427e" + "sample1.bam:md5,d41d8cd98f00b204e9800998ecf8427e", + "sample1.bam.bai:md5,d41d8cd98f00b204e9800998ecf8427e" ] ], "3": [ @@ -44,7 +46,7 @@ { "id": "sample1" }, - "sample1.cnv_metrics.csv:md5,d41d8cd98f00b204e9800998ecf8427e" + "sample1.wgs_coverage_metrics.csv:md5,d41d8cd98f00b204e9800998ecf8427e" ] ], "4": [ @@ -52,7 +54,7 @@ { "id": "sample1" }, - "sample1.mapping_metrics.csv:md5,d41d8cd98f00b204e9800998ecf8427e" + "sample1.cnv_metrics.csv:md5,d41d8cd98f00b204e9800998ecf8427e" ] ], "5": [ @@ -60,7 +62,7 @@ { "id": "sample1" }, - "sample1.ploidy_estimation_metrics.csv:md5,d41d8cd98f00b204e9800998ecf8427e" + "sample1.mapping_metrics.csv:md5,d41d8cd98f00b204e9800998ecf8427e" ] ], "6": [ @@ -68,16 +70,33 @@ { "id": "sample1" }, - "sample1.gvcf_metrics.csv:md5,d41d8cd98f00b204e9800998ecf8427e" + "sample1.ploidy_estimation_metrics.csv:md5,d41d8cd98f00b204e9800998ecf8427e" ] ], "7": [ + [ + { + "id": "sample1" + }, + "sample1.gvcf_metrics.csv:md5,d41d8cd98f00b204e9800998ecf8427e" + ] + ], + "8": [ [ "DRAGEN", "dragen", "bash: line 1: dragen: command not found" ] ], + "bam_bai": [ + [ + { + "id": "sample1" + }, + "sample1.bam:md5,d41d8cd98f00b204e9800998ecf8427e", + "sample1.bam.bai:md5,d41d8cd98f00b204e9800998ecf8427e" + ] + ], "cnv_metrics": [ [ { @@ -123,6 +142,7 @@ }, [ "sample1.bam:md5,d41d8cd98f00b204e9800998ecf8427e", + "sample1.bam.bai:md5,d41d8cd98f00b204e9800998ecf8427e", "sample1.cnv_metrics.csv:md5,d41d8cd98f00b204e9800998ecf8427e", "sample1.gvcf_metrics.csv:md5,d41d8cd98f00b204e9800998ecf8427e", "sample1.mapping_metrics.csv:md5,d41d8cd98f00b204e9800998ecf8427e", @@ -165,10 +185,10 @@ ] } ], - "timestamp": "2026-08-18T11:08:58.003281", + "timestamp": "2026-08-28T16:31:56.534107", "meta": { "nf-test": "0.9.5", - "nextflow": "25.10.2" + "nextflow": "26.04.6" } } } \ No newline at end of file diff --git a/modules/UMCUGenetics/untar/genome/main.nf b/modules/UMCUGenetics/untar/genome/main.nf index 30f5d51..4c989f0 100644 --- a/modules/UMCUGenetics/untar/genome/main.nf +++ b/modules/UMCUGenetics/untar/genome/main.nf @@ -1,5 +1,5 @@ process UNTAR_GENOME { - tag "${archive}" + tag "${meta.id}" label 'process_single' container "${workflow.containerEngine in ['singularity', 'apptainer'] && !task.ext.singularity_pull_docker_container @@ -7,18 +7,40 @@ process UNTAR_GENOME { : 'community.wave.seqera.io/library/coreutils_grep_gzip_lbzip2_pruned:838ba80435a629f8'}" input: - tuple val(meta), path(archive) + tuple val(meta), path(genome_tar) output: - tuple val(meta), path('genome'), emit: untar + tuple val(meta), path('dragen_ref'), emit: dragen_ref + tuple val(meta), path('genome.fa'), emit: fasta + tuple val(meta), path("genes.gtf.gz"), emit: gtf tuple val("${task.process}"), val('UNTAR'), eval('echo 1.0.0'), emit: versions_untar_genome, topic: versions when: task.ext.when == null || task.ext.when script: + def hashtable_version = params.dragen_hashtable_version ?: '' """ - mkdir genome - tar -xavf ${archive} -C genome + tar -xzf ${genome_tar} + mapfile -t hts < <(find . -name hash_table.cfg -printf '%h\\n' | grep -E '/[0-9]+\$' | sort -V) + + if [ \${#hts[@]} -eq 0 ]; then + echo "ERROR: no DRAGEN hashtable (hash_table.cfg) found in ${genome_tar}" >&2 + exit 1 + fi + + + if [ -n "${hashtable_version}" ]; then + ref_dir=\$(printf '%s\\n' "\${hts[@]}" | grep -E "/${hashtable_version}\$") || { + echo "ERROR: hashtable v${hashtable_version} not found. Available: \${hts[*]}" >&2 + exit 1 + } + else + ref_dir=\${hts[-1]} # Highest version + fi + + echo "Using DRAGEN ref-dir: \$ref_dir" >&2 + ln -s "\$(realpath \$ref_dir)" dragen_ref + """ } diff --git a/modules/UMCUGenetics/untar/genome/tests/main.nf.test b/modules/UMCUGenetics/untar/genome/tests/main.nf.test index cf588e8..0ec417d 100644 --- a/modules/UMCUGenetics/untar/genome/tests/main.nf.test +++ b/modules/UMCUGenetics/untar/genome/tests/main.nf.test @@ -15,7 +15,7 @@ nextflow_process { """ input[0] = channel.of([ [id: 'genome'], - file(params.subworkflows_umcu_testdata_base_path + "prepare_ica_references/genome.tar", checkIfExists: true) + file(params.subworkflows_umcu_testdata_base_path + "prepare_ica_references/genome.tar.gz", checkIfExists: true) ]) """ } @@ -23,6 +23,9 @@ nextflow_process { then{ assertAll( { assert process.success }, + { assert path(process.out.dragen_ref[0][1]).exists() }, + { assert path(process.out.fasta[0][1]).exists() }, + { assert path(process.out.gtf[0][1]).exists() }, { assert snapshot( process.out.findAll { key, val -> key.startsWith('versions') } ).match() } diff --git a/modules/UMCUGenetics/untar/genome/tests/main.nf.test.snap b/modules/UMCUGenetics/untar/genome/tests/main.nf.test.snap index 83357a9..1c01a83 100644 --- a/modules/UMCUGenetics/untar/genome/tests/main.nf.test.snap +++ b/modules/UMCUGenetics/untar/genome/tests/main.nf.test.snap @@ -11,7 +11,7 @@ ] } ], - "timestamp": "2026-08-21T11:44:44.991954", + "timestamp": "2026-08-28T10:56:51.250213", "meta": { "nf-test": "0.9.5", "nextflow": "26.04.6" diff --git a/subworkflows/UMCUGenetics/bam_prs/nextflow.config b/subworkflows/UMCUGenetics/bam_prs/nextflow.config index 63e1a31..6f2cb0d 100644 --- a/subworkflows/UMCUGenetics/bam_prs/nextflow.config +++ b/subworkflows/UMCUGenetics/bam_prs/nextflow.config @@ -44,7 +44,6 @@ process { cpus = 2 memory = { 10.GB * task.attempt } time = { 30.m * task.attempt } - array = params.job_array_size ext.args = { [ "--allow-extra-chr", diff --git a/subworkflows/UMCUGenetics/prepare_ica_references/main.nf b/subworkflows/UMCUGenetics/prepare_ica_references/main.nf index 394ad2a..f93b821 100644 --- a/subworkflows/UMCUGenetics/prepare_ica_references/main.nf +++ b/subworkflows/UMCUGenetics/prepare_ica_references/main.nf @@ -11,23 +11,18 @@ workflow PREPARE_ICA_REFERENCES { UNTAR_GENOME(genome_tar) - genome_fasta = UNTAR_GENOME.out.untar - .map{ meta, dir -> [meta, file("${dir}/genome.fa", - checkIfExists: !workflow.stubRun) - ]} - SAMTOOLS_FAIDX( - genome_fasta.map {meta, fasta -> [meta, fasta, []]}, + UNTAR_GENOME.out.fasta.map {meta, fasta -> [meta, fasta, []]}, false ) - GATK4_CREATESEQUENCEDICTIONARY(genome_fasta) + GATK4_CREATESEQUENCEDICTIONARY(UNTAR_GENOME.out.fasta) emit: - genome_fasta = genome_fasta + genome_fasta = UNTAR_GENOME.out.fasta genome_fai = SAMTOOLS_FAIDX.out.fai genome_dict = GATK4_CREATESEQUENCEDICTIONARY.out.dict - genome_dir = UNTAR_GENOME.out.untar + genome_dir = UNTAR_GENOME.out.dragen_ref } diff --git a/subworkflows/UMCUGenetics/prepare_ica_references/tests/main.nf.test b/subworkflows/UMCUGenetics/prepare_ica_references/tests/main.nf.test index 8cb92b8..acbe415 100644 --- a/subworkflows/UMCUGenetics/prepare_ica_references/tests/main.nf.test +++ b/subworkflows/UMCUGenetics/prepare_ica_references/tests/main.nf.test @@ -24,12 +24,13 @@ nextflow_workflow { } test("Validate genome untar and processing") { + options "-stub" when{ workflow{ """ input[0] = channel.of([ [id: 'genome'], - file(params.subworkflows_umcu_testdata_base_path + "prepare_ica_references/genome.tar", checkIfExists: true) + file(params.subworkflows_umcu_testdata_base_path + "prepare_ica_references/genome.tar.gz", checkIfExists: true) ]) """ } @@ -37,11 +38,10 @@ nextflow_workflow { then { assertAll( { assert workflow.success }, - { assert snapshot( - workflow.out.genome_fasta, - workflow.out.genome_fai, - workflow.out.genome_dict - ).match()} + { assert path(workflow.out.genome_fasta[0][1]).exists() }, + { assert path(workflow.out.genome_fai[0][1]).exists() }, + { assert path(workflow.out.genome_dict[0][1]).exists() }, + { assert path(workflow.out.genome_dir[0][1]).exists() }, ) } diff --git a/subworkflows/UMCUGenetics/prepare_ica_references/tests/main.nf.test.snap b/subworkflows/UMCUGenetics/prepare_ica_references/tests/main.nf.test.snap deleted file mode 100644 index 351f20e..0000000 --- a/subworkflows/UMCUGenetics/prepare_ica_references/tests/main.nf.test.snap +++ /dev/null @@ -1,35 +0,0 @@ -{ - "Validate genome untar and processing": { - "content": [ - [ - [ - { - "id": "genome" - }, - "genome.fa:md5,d566f7aeef35edc9da9d38d672f34fa9" - ] - ], - [ - [ - { - "id": "genome" - }, - "genome.fa.fai:md5,1211e0613e9e88446fc1b2b0222a3339" - ] - ], - [ - [ - { - "id": "genome" - }, - "genome.dict:md5,b8b7b776dfdefe7badf859248af14525" - ] - ] - ], - "timestamp": "2026-08-19T13:27:32.222553", - "meta": { - "nf-test": "0.9.5", - "nextflow": "26.04.6" - } - } -} \ No newline at end of file