Includes STRs as a data source

.pre-commit-config.yaml

@@ -28,7 +28,7 @@ repos:
 
   # Static type analysis (as much as it's possible in python using type hints)
   - repo: https://github.com/pre-commit/mirrors-mypy
-    rev: v1.20.2
+    rev: v2.1.0
     hooks:
       - id: mypy
         args: [--pretty, --show-error-codes, --install-types, --non-interactive]

CHANGELOG.md

@@ -28,6 +28,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
   * Talos previously only logged results, not rejections. This logs misses, and explanations e.g. threshold failure, family test, insufficient read depth, comp-het with only support categories
   * Using the "confidence_level" configuration setting can allow PanelApp genes with lower evidence levels (1 >= Red, 2 >= Amber) to be used in analysis. The default level remains 3/Green-only.
   * The HTML report shows, for each gene, the panels it was found in, and the confidence level associated evidence level on each panel. Checkboxes can be used to filter to specific confidence levels.
+  * STR data can now be handled. This is not currently implemented in Nextflow, but could be if there's appetite outside CPG.
 
 ### Changed
 

Dockerfile

@@ -61,7 +61,7 @@ COPY --from=bcftools_compiler /bcftools_install/usr/local/lib/ /usr/local/lib/
 RUN ldconfig
 
 ARG ECHTVAR_VERSION=v0.2.2
-ARG VERSION=11.0.3
+ARG VERSION=11.1.0
 
 RUN wget -q -O /bin/echtvar "https://github.com/brentp/echtvar/releases/download/${ECHTVAR_VERSION}/echtvar" && \
     chmod +x /bin/echtvar

README.md

@@ -69,7 +69,7 @@ There are two primary workflows:
 To build the Docker image:
 
 ```
-docker build -t talos:11.0.3 .
+docker build -t talos:11.1.0 .
 ```
 
 ### **2. Download Annotation Resources**

docs/getting-started.md

@@ -25,7 +25,7 @@ You will need:
 Build the Talos Docker image locally:
 
 ```bash
-docker build -t talos:11.0.3 .
+docker build -t talos:11.1.0 .
 ```
 
 ---
@@ -81,7 +81,10 @@ The optional columns (history, ext_ids, seqr_map, mito) can be omitted completel
 | `seqr_map` | optional | ID mapping for Seqr hyperlinks in the HTML report.                             |
 | `mito`     | optional | Path to a joint-called mitochondrial VCF.                                      |
 
-### Input types
+STR can be handled by Talos, but only data called by STRipy, and aggregated into a joint VCF format by the script `talos/scripts/stripy_json_to_vcf.py`.
+This is not yet exposed in the nextflow implementation, but may be in future.
+
+### Small-variant VCF input types
 
 <table>
     <colgroup>

nextflow.config

@@ -16,8 +16,8 @@ params {
     large_files = "large_files"
     ref_genome = "${params.large_files}/ref.fa"
 
-    // Docker container - "docker build -t talos:11.0.3 ."
-    container = 'talos:11.0.3'
+    // Docker container - "docker build -t talos:11.1.0 ."
+    container = 'talos:11.1.0'
 
     // MANE transcript resource
     mane = "${params.large_files}/MANE.GRCh38.v1.5.summary.txt.gz"
@@ -31,6 +31,7 @@ params {
     // Result files from the CreateRoiFromGff3 nextflow module
     ensembl_bed = "${params.processed_annotations}/GRCh38.bed"
     ensembl_merged_bed = "${params.processed_annotations}/GRCh38_merged.bed"
+    ensembl_symbol_lookup = "${params.processed_annotations}/GRCh38_symbol_to_ensg.json"
 
     // AlphaMissense raw data, and reformatted as an echtvar-compatible zip
     alphamissense_tsv = "${params.large_files}/AlphaMissense_hg38.tsv.gz"

nextflow/inputs/config.toml

@@ -12,6 +12,12 @@ forced_panels = [ 144, 239]
 # 1 = Red, Amber, or Green
 confidence_level = 1
 
+# confidence level - set this to a lower value to allow non-green PanelApp genes into the analysis
+# 3 = Green only
+# 2 = Amber or Green only
+# 1 = Red, Amber, or Green
+confidence_level = 3
+
 [[GeneratePanelData.manual_overrides]]
 # this section permits the manual addition of genes to the panel data
 # each gene here will be folded into the panelapp data after standard API queries have taken place

nextflow/modules/prep/CreateRoiFromGff3/main.nf

@@ -8,9 +8,11 @@ process CreateRoiFromGff3 {
     // generates a BED file one-gene-per-line, for per-gene annotations
     // generates an BED with overlapping intervals merged for filtering
     // sorts both
+    // also generates a JSON file mapping symbols to ENSG IDs
     output:
         path "GRCh38.bed", emit: bed
         path "GRCh38_merged.bed", emit: merged_bed
+        path "GRCh38_symbol_to_ensg.json", emit: json_lookup
 
     script:
         """
@@ -19,7 +21,8 @@ process CreateRoiFromGff3 {
         python -m talos.annotation_scripts.create_roi_from_gff3 \
             --gff3 ${gff} \
             --unmerged_output unsorted_GRCh38.bed \
-            --merged_output unsorted_merged_GRCh38.bed
+            --merged_output unsorted_merged_GRCh38.bed \
+            --json_output GRCh38_symbol_to_ensg.json
 
         sort -k1,1V -k2,2n unsorted_GRCh38.bed > GRCh38.bed
         sort -k1,1V -k2,2n unsorted_merged_GRCh38.bed > GRCh38_merged.bed

nextflow_schema.json

@@ -33,7 +33,7 @@
         },
         "container": {
           "type": "string",
-          "default": "talos:11.0.3",
+          "default": "talos:11.1.0",
           "description": "Docker container used by processes."
         },
         "mane": {

preparation.nf

@@ -121,13 +121,15 @@ workflow {
     // generate the Region-of-interest BED file from Ensembl GFF3
     // generates a per-gene BED file with ID annotations
     // and a overlap-merged version of the same for more efficient region filtering
-    if (!file(params.ensembl_bed).exists() || !file(params.ensembl_merged_bed).exists()) {
+    if (!file(params.ensembl_bed).exists() || !file(params.ensembl_merged_bed).exists() || !file(params.ensembl_symbol_lookup).exists()) {
         CreateRoiFromGff3(ch_gff)
         ch_bed = CreateRoiFromGff3.out.bed
         ch_merged_bed = CreateRoiFromGff3.out.merged_bed
+        ch_symbol_lookup = CreateRoiFromGff3.out.json_lookup
     } else {
         ch_merged_bed = channel.fromPath(params.ensembl_merged_bed, checkIfExists: true)
         ch_bed = channel.fromPath(params.ensembl_bed, checkIfExists: true)
+        ch_symbol_lookup = channel.fromPath(params.ensembl_symbol_lookup, checkIfExists: true)
     }
 
     // pull and parse the MANE data into a Hail Table
@@ -156,6 +158,7 @@ workflow {
         alphamissense = ch_alphamissense_zip
         bed = ch_bed
         merged_bed = ch_merged_bed
+        symbol_lookup = ch_symbol_lookup
         clinvar_all = ch_clinvar_all
         clinvar_pm5 = ch_clinvar_pm5
         mitimpact = ch_mitimpact_zip
@@ -172,6 +175,8 @@ output {
     }
     merged_bed {
     }
+    symbol_lookup {
+    }
     clinvar_all {
     }
     clinvar_pm5 {

pyproject.toml

@@ -6,7 +6,7 @@ build-backend = "hatchling.build"
 name='talos'
 description='Centre for Population Genomics Variant Prioritisation'
 readme = "README.md"
-version="11.0.3"
+version="11.1.0"
 requires-python = ">=3.10,<3.12"
 license-files = ["LICENSE"]
 classifiers=[
@@ -57,6 +57,9 @@ docs = [
     'mkdocs-material>=9.5',
     'mkdocs-include-markdown-plugin>=6.2',
 ]
+stripy = [
+    'pysam'
+]
 
 [project.urls]
 Repository = "https://github.com/populationgenomics/talos"
@@ -102,7 +105,8 @@ ignore = [
     "PLR0913", # Too many arguments in function (> 5)
     "C901", # method is too complex (> 10 conditions)
     "S311", # Standard pseudo-random generators are not suitable for cryptographic purposes
-    "PLR2004" #  Magic value used in comparison, consider replacing `X` with a constant variable. I don't rate this
+    "PLR2004", # Magic value used in comparison, consider replacing `X` with a constant variable. I don't rate this
+    "PLW0108", # Lambda may be unnecessary; consider inlining inner function
 ]
 
 [tool.ruff.format]
@@ -127,7 +131,7 @@ hail = ["hail", "hailtop"]
 "test/test_cpg_flow_utils.py" = ['ANN202', 'ARG001']
 
 [tool.bumpversion]
-current_version = "11.0.3"
+current_version = "11.1.0"
 parse = "(?P<major>\\d+)\\.(?P<minor>\\d+)\\.(?P<patch>\\d+)"
 serialize = ["{major}.{minor}.{patch}"]
 commit = true

src/talos/annotation_scripts/create_roi_from_gff3.py

@@ -9,6 +9,7 @@
 """
 
 import gzip
+import json
 import re
 from argparse import ArgumentParser
 
@@ -31,29 +32,43 @@
 CANONICAL_CONTIGS = [f'chr{x}' for x in list(range(1, 23))] + ['chrX', 'chrY', 'chrM']
 
 
-def main(gff3_file: str, unmerged_output: str, merged_output: str, flanking: int = 2000):
+def main(
+    gff3_file: str,
+    unmerged_output: str,
+    merged_output: str,
+    flanking: int = 2000,
+    json_output: str | None = None,
+):
     """
     Read the GFF3 file, and generate a BED file of gene regions, plus padding
     Args:
         gff3_file (str): path to the GFF3 file
         unmerged_output (str): path to the intended BED output file
         merged_output (str): path to generate a BED file with merged overlapping rows
         flanking (int): number of bases to add before and after each gene
+        json_output (str | None): optional, path to write a Symbol: ENSG dictionary
     """
 
-    unmerged_lines = generate_bed_lines(gff3_file, unmerged_output, flanking)
+    unmerged_lines, as_dict = generate_bed_lines(gff3_file, unmerged_output, flanking)
     merge_output(unmerged_lines, merged_output)
 
+    # if a json version was requested, write that out
+    if json_output is not None:
+        with open(json_output, 'w') as write_handle:
+            json.dump(as_dict, write_handle, indent=4)
+
 
 def generate_bed_lines(
     gff3_file: str,
     output: str,
     flanking: int = FLANKING_REGION,
-) -> list[tuple[str, int, int]]:
+) -> tuple[list[tuple[str, int, int]], dict[str, str]]:
     """
     Generate the new BED file, and return the lines as a list of lists for merging.
     """
     output_lines: list[tuple[str, int, int]] = []
+    output_as_dict: dict[str, str] = {}
+
     # open and iterate over the GFF3 file
     with gzip.open(gff3_file, 'rt') as handle, open(output, 'w') as write_handle:
         for line in handle:
@@ -85,6 +100,8 @@ def generate_bed_lines(
                 print(f'Failed to extract gene name from {line_as_list[DETAILS_INDEX]}')
                 continue
 
+            output_as_dict[gene_name] = gene_id
+
             # write the line to the output
             output_list = [
                 f'chr{line_as_list[CHROM_INDEX]}',
@@ -100,7 +117,7 @@ def generate_bed_lines(
                     int(line_as_list[END_INDEX]) + flanking,
                 ),
             )
-    return output_lines
+    return output_lines, output_as_dict
 
 
 def merge_output(
@@ -166,12 +183,18 @@ def cli_main():
         help='Regions to add to each gene',
         default=FLANKING_REGION,
     )
+    parser.add_argument(
+        '--json_output',
+        help='Write a {Gene Symbol: ENSG, } dictionary in JSON format.',
+        required=False,
+    )
     args = parser.parse_args()
     main(
         gff3_file=args.gff3,
         unmerged_output=args.unmerged_output,
         merged_output=args.merged_output,
         flanking=args.flanking,
+        json_output=args.json_output,
     )
 
 

src/talos/cpg_internal_scripts/CPG_Dockerfile

@@ -16,7 +16,7 @@ FROM base AS bcftools_compiler
 
 # BCFtools 1.22+ doesn't do any inference on chr-prefixes during CSQ, so the mismatch between fasta/vcf, and the gff3 (no chr-prefix) needs to be handled explicitly
 # BCFtools >=1.22 is required internally as it's the first version with a built-in Mitochondrial lookup table.
-# AS OF 11.0.3, Talos is building BCFtools from a private fork. This fork contains a single change - csq applies annotations
+# AS OF 11.1.0, Talos is building BCFtools from a private fork. This fork contains a single change - csq applies annotations
 # to both coding and non-coding genes in the event of overlapping genes. By default BCFtools skips non-coding gene annotation
 # if a coding transcript consequence was detected, but in practice this is masking clinically relevant non-coding gene variation
 # in cases where the non-coding gene overlaps with a non-clinically relevant coding gene.
@@ -53,7 +53,7 @@ RUN apt-get update && apt-get install --no-install-recommends -y \
 FROM base AS talos
 
 ARG ECHTVAR_VERSION=v0.2.2
-ARG VERSION=11.0.3
+ARG VERSION=11.1.0
 
 COPY --from=bcftools_compiler /bcftools_install/usr/local/bin/* /usr/local/bin/
 COPY --from=bcftools_compiler /bcftools_install/usr/local/libexec/bcftools/* /usr/local/libexec/bcftools/

src/talos/cpg_internal_scripts/talos_stages.py

@@ -628,6 +628,26 @@ def queue_jobs(self, cohort: targets.Cohort, inputs: stage.StageInput) -> stage.
             )['vcf.bgz']
             mito_vcf_arg = f'--labelled_mito {mito_vcf} '
 
+        # is this run going to include STR data?
+        str_vcf_arg = ''
+        # screen out cohorts/datasets which we don't want to run STRs for
+        str_cohort = cohort.dataset.name in config.config_retrieve(['workflow', 'str_cohorts'], [])
+        if str_cohort and (
+            str_vcf := query_for_latest_analysis(
+                dataset=cohort.dataset.name,
+                analysis_type='vcf',
+                long_read=config.config_retrieve(['workflow', 'long_read'], False),
+                stage_name='MakeStripyJointCall',
+            )
+        ):
+            stripy_vcf = hail_batch.get_batch().read_input_group(
+                **{
+                    'vcf.bgz': str_vcf,
+                    'vcf.bgz.tbi': f'{str_vcf}.tbi',
+                },
+            )['vcf.bgz']
+            str_vcf_arg = f'--str {stripy_vcf} '
+
         labelled_vcf = hail_batch.get_batch().read_input_group(
             **{
                 'vcf.bgz': hail_inputs,
@@ -657,7 +677,7 @@ def queue_jobs(self, cohort: targets.Cohort, inputs: stage.StageInput) -> stage.
                 --labelled_vcf {labelled_vcf} \\
                 --output {job.output} \\
                 --panelapp {panelapp_data} \\
-                --pedigree {pedigree} {sv_vcf_arg} {history_string} {mito_vcf_arg}
+                --pedigree {pedigree} {sv_vcf_arg} {history_string} {mito_vcf_arg} {str_vcf_arg}
             """,
         )
         expected_out = self.expected_outputs(cohort)

src/talos/create_talos_html.py

@@ -23,7 +23,16 @@
 from loguru import logger
 
 from talos.config import config_retrieve
-from talos.models import PanelApp, PanelDetail, PanelShort, ReportVariant, ResultData, SmallVariant, StructuralVariant
+from talos.models import (
+    PanelApp,
+    PanelDetail,
+    PanelShort,
+    ReportVariant,
+    ResultData,
+    ShortTandemRepeat,
+    SmallVariant,
+    StructuralVariant,
+)
 from talos.utils import read_json_from_path
 
 JINJA_TEMPLATE_DIR = Path(__file__).absolute().parent / 'templates'
@@ -157,7 +166,7 @@ def variant_in_forbidden_gene(variant_obj: ReportVariant, forbidden_genes):
         if gene_id in forbidden_genes:
             return True
 
-    if isinstance(variant_obj.var_data, StructuralVariant):
+    if isinstance(variant_obj.var_data, (ShortTandemRepeat, StructuralVariant)):
         return False
 
     # Allow for exclusion by Symbol too
@@ -568,6 +577,11 @@ def get_var_change(self) -> str:
         if isinstance(self.var_data, StructuralVariant):
             return f'{self.var_data.info["svtype"]} {self.var_data.info["svlen"]}bp'
 
+        if isinstance(self.var_data, ShortTandemRepeat):
+            locus = self.var_data.locus or self.var_data.info.get('locus')
+            repeats = ', '.join(map(str, self.var_data.info['sample_repeats']))
+            return f'STR {locus}, Repeats: {repeats}'
+
         raise ValueError(f'Unknown variant type: {self.var_data.__class__.__name__}')
 
     def __init__(self, report_variant: ReportVariant, sample: Sample, ext_labels: list, html_builder: HTMLBuilder):  # noqa: PLR0915