Includes STRs as a data source (#667)

.github/workflows/build_deploy_main.yaml

@@ -3,6 +3,7 @@ on:
   push:
     branches:
       - main
+      - dev
 
 permissions:
   id-token: write
@@ -18,7 +19,7 @@ jobs:
     runs-on: ubuntu-latest
     environment: production
     outputs:
-      matrix: ${{ steps.get_next_version.outputs.next_version }}
+      next_version: ${{ steps.get_next_version.outputs.next_version }}
     steps:
       - name: 'Checkout repo'
         uses: actions/checkout@v4
@@ -55,13 +56,12 @@ jobs:
     environment: production
     needs:
       - get_next_version
-    if: ${{ needs.get_next_version.outputs.matrix != '{"include":[]}' && needs.get_next_version.outputs.matrix != '' }}
-    strategy:
-      matrix: ${{ fromJson(needs.get_next_version.outputs.matrix) }}
+    if: ${{ needs.get_next_version.outputs.next_version != '' && github.event.pull_request.draft == false }}
     env:
       DOCKER_BUILDKIT: 1
       BUILDKIT_PROGRESS: plain
       CLOUDSDK_CORE_DISABLE_PROMPTS: 1
+      TAG: ${{ needs.get_next_version.outputs.next_version }}${{ github.ref == 'refs/heads/dev' && '_dev' || '' }}
     steps:
       - uses: actions/checkout@v4
         with:
@@ -84,9 +84,8 @@ jobs:
       - name: 'build image'
         run: |
           docker build . -f src/talos/cpg_internal_scripts/CPG_Dockerfile \
-          --build-arg VERSION=${{ matrix.tag }} \
-          --tag ${{ env.IMAGES_PREFIX }}/${{ matrix.name }}:${{ matrix.tag }}
+          --tag ${{ env.IMAGES_PREFIX }}/talos:${{ env.TAG }}
 
       - name: 'push image'
         run: |
-          docker push "${{ env.IMAGES_PREFIX }}/${{ matrix.name }}:${{ matrix.tag }}"
+          docker push "${{ env.IMAGES_PREFIX }}/talos:${{ env.TAG }}"

.github/workflows/dev_auto_build.yaml

@@ -24,7 +24,7 @@ jobs:
     runs-on: ubuntu-latest
     environment: production
     outputs:
-      matrix: ${{ steps.get_next_version.outputs.next_version }}
+      next_version: ${{ steps.get_next_version.outputs.next_version }}
     steps:
       - name: 'Checkout repo'
         uses: actions/checkout@v4
@@ -61,9 +61,7 @@ jobs:
     environment: production
     needs:
       - get_next_version
-    if: ${{ needs.get_next_version.outputs.matrix != '{"include":[]}' && needs.get_next_version.outputs.matrix != '' && github.event.pull_request.draft == false }}
-    strategy:
-      matrix: ${{ fromJson(needs.get_next_version.outputs.matrix) }}
+    if: ${{ needs.get_next_version.outputs.next_version != '' && github.event.pull_request.draft == false }}
     steps:
       - name: 'checkout repo'
         uses: actions/checkout@v4
@@ -88,12 +86,11 @@ jobs:
       - name: 'build image with get_next_version'
         run: |
           docker build . -f src/talos/cpg_internal_scripts/CPG_Dockerfile \
-          --build-arg VERSION=${{ matrix.tag }} \
-          --tag ${{ env.IMAGES_PREFIX }}/${{ matrix.name }}:${{ matrix.tag }}
+          --tag ${{ env.IMAGES_PREFIX }}/talos:${{ needs.get_next_version.outputs.next_version }}
 
       - name: 'Push to dev artifactory'
         run: |
-          docker push "${{ env.IMAGES_PREFIX }}/${{ matrix.name }}:${{ matrix.tag }}" | tee push.log
+          docker push "${{ env.IMAGES_PREFIX }}/talos:${{ needs.get_next_version.outputs.next_version }}" | tee push.log
           digest=$(grep 'digest:' push.log | grep -o 'sha256:[0-9a-f]*')
           echo ":package: Link to image:" >> $GITHUB_STEP_SUMMARY
-          echo "https://console.cloud.google.com/artifacts/docker/cpg-common/australia-southeast1/images-dev/${{ matrix.name }}/$digest?project=cpg-common" >> $GITHUB_STEP_SUMMARY
+          echo "https://console.cloud.google.com/artifacts/docker/cpg-common/australia-southeast1/images-dev/talos/$digest?project=cpg-common" >> $GITHUB_STEP_SUMMARY

.github/workflows/get_version.py

@@ -1,5 +1,4 @@
 import json
-import logging
 import os
 import re
 import subprocess
@@ -9,11 +8,11 @@
 def extract_version_from_file(file_path: str) -> str | None:
     """
     Extract the version from a Dockerfile by searching for a line like:
-    ARG VERSION=1.0.0
+    ENV VERSION=1.0.0
     """
     with open(file_path) as f:
         content = f.read()
-    pattern = re.compile(r'^\s*ARG\s+VERSION\s*=\s*([^\s]+)', re.MULTILINE)
+    pattern = re.compile(r'^\s*ENV\s+VERSION\s*=\s*([^\s]+)', re.MULTILINE)
     match = pattern.search(content)
     return match.group(1) if match else None
 
@@ -27,48 +26,31 @@ def get_next_version_tag(folder: str, version: str) -> str:
         'GCP_BASE_IMAGE',
         'australia-southeast1-docker.pkg.dev/cpg-common/images',
     )
-    base_image_path_archive = os.environ.get(
-        'GCP_BASE_ARCHIVE_IMAGE',
-        'australia-southeast1-docker.pkg.dev/cpg-common/images-archive',
-    )
-    full_image_name_prod = f'{base_image_path_prod}/{folder}'
-    full_image_name_archive = f'{base_image_path_archive}/{folder}'
-
-    tags_list = []
-
-    logging.basicConfig(level=logging.ERROR)
-
-    for full_image_name in [full_image_name_prod, full_image_name_archive]:
-        cmd = [
-            'gcloud',
-            'container',
-            'images',
-            'list-tags',
-            full_image_name,
-            '--format=json',
-        ]
-        try:
-            result = subprocess.run(cmd, capture_output=True, text=True, check=True)  # noqa: S603
-        except subprocess.CalledProcessError:
-            logging.error(f'Failed to list tags for {full_image_name}')
-            continue
-
-        # If existing tags are found, proceed to determine the next version.
-        tags_list += json.loads(result.stdout)
+    cmd = [
+        'gcloud',
+        'container',
+        'images',
+        'list-tags',
+        f'{base_image_path_prod}/{folder}',
+        '--format=json',
+    ]
 
     max_suffix = 0
     pattern = re.compile(rf'^{re.escape(version)}-(\d+)$')
 
-    # If no tags are found, it returns the next version as -1.
-    for entry in tags_list:
-        tags = entry.get('tags', [])
-        for tag in tags:
-            match = pattern.match(tag)
+    try:
+        result = subprocess.run(cmd, capture_output=True, text=True, check=True)  # noqa: S603
+        for block in json.loads(result.stdout):
+            if not block.get('tags', []):
+                continue
+            match = pattern.match(block['tags'][0])
             if match:
                 num = int(match.group(1))
                 max_suffix = max(max_suffix, num)
-    new_suffix = max_suffix + 1
-    return f'{version}-{new_suffix}'
+    except subprocess.CalledProcessError as err:
+        raise RuntimeError('Failed to list tags for the given image') from err
+
+    return f'{version}-{max_suffix + 1}'
 
 
 def main():
@@ -77,17 +59,13 @@ def main():
     current_version = extract_version_from_file(dockerfile_name)
     if current_version is None:
         # Throw an error here
-        raise NotImplementedError('The Dockerfile needs to contain a version string in the format "ARG VERSION=x.x.x"')
+        raise NotImplementedError('The Dockerfile needs to contain a version string in the format "ENV VERSION=x.x.x"')
 
     # Determine the next available tag based on current_version.
     new_tag = get_next_version_tag(container_name, current_version)
 
-    include_entries = [{'name': container_name, 'tag': new_tag}]
-
-    # Build the final matrix structure.
-    matrix = {'include': include_entries}
-    print(json.dumps(matrix, separators=(',', ':')), file=sys.stderr)
-    print(json.dumps(matrix, separators=(',', ':')), end='')
+    print(new_tag, file=sys.stderr)
+    print(new_tag, end='')
 
 
 main()

.pre-commit-config.yaml

@@ -28,7 +28,7 @@ repos:
 
   # Static type analysis (as much as it's possible in python using type hints)
   - repo: https://github.com/pre-commit/mirrors-mypy
-    rev: v1.20.2
+    rev: v2.1.0
     hooks:
       - id: mypy
         args: [--pretty, --show-error-codes, --install-types, --non-interactive]

CHANGELOG.md

@@ -17,6 +17,19 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 <!--changelog-start-->
 <!--latest-start-->
 
+[11.1.0] - 2026-07
+
+### Removed
+
+  * All references to the private build of BCFtools. the `--greedy` behaviour is now released as standard in BCFtools 1.24
+
+### Added
+
+  * STR functionality! This is currently specific to STRipy data, with a script to translate STRipy JSON data into a pseudo-joint called VCF
+  * If there is appetite for STR functionality, this could be expanded to other STR callers.
+
+<!--latest-end-->
+
 [11.0.0] - 2026-05-15
 
 ### Added
@@ -48,8 +61,6 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 > NOTE! For existing Talos users, obtaining this extra annotation will require the re-run of the annotation workflow. One example gene which is known to be obscured by this default behaviour was RNU2-2, but there may be others.
 
-<!--latest-end-->
-
 [10.0.1] - 2026-03-23
 
 ### Fixed

Dockerfile

@@ -19,49 +19,35 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
 
 FROM base AS bcftools_compiler
 
-ARG BCFTOOLS_VERSION=1.23.1
-
-# AS OF 11.0.0, Talos is building BCFtools from a private fork. This fork contains a single change - csq applies annotations
-# to both coding and non-coding genes in the event of overlapping genes. By default BCFtools skips non-coding gene annotation
-# if a coding transcript consequence was detected, but in practice this is masking clinically relevant non-coding gene variation
-# in cases where the non-coding gene overlaps with a non-clinically relevant coding gene.
-# The change made is to always search for non-coding variation, but to mask non-coding consequences in otherwise coding transcripts
-# if a coding change was already detected.
+ARG BCFTOOLS_VERSION=1.24
+
 RUN apt-get update && apt-get install --no-install-recommends -y \
-    autoconf \
     gcc \
-    git \
     libbz2-dev \
     libcurl4-openssl-dev \
     liblzma-dev \
     libssl-dev \
     make \
     zlib1g-dev && \
-    wget https://github.com/samtools/htslib/releases/download/${BCFTOOLS_VERSION}/htslib-${BCFTOOLS_VERSION}.tar.bz2 && \
-    tar -xf htslib-${BCFTOOLS_VERSION}.tar.bz2 && \
-    cd htslib-${BCFTOOLS_VERSION} && \
-    ./configure --enable-libcurl && \
-    make && \
-    make DESTDIR=/bcftools_install install && \
-    cd .. && \
-    git clone https://github.com/populationgenomics/bcftools.git && \
-    cd bcftools && \
-    autoheader && \
-    autoconf && \
+    rm -rf /var/lib/apt/lists/* && \
+    wget https://github.com/samtools/bcftools/releases/download/${BCFTOOLS_VERSION}/bcftools-${BCFTOOLS_VERSION}.tar.bz2 && \
+    tar -xf bcftools-${BCFTOOLS_VERSION}.tar.bz2 && \
+    cd bcftools-${BCFTOOLS_VERSION} && \
     ./configure --enable-libcurl --enable-s3 --enable-gcs && \
     make && \
     strip bcftools plugins/*.so && \
+    make DESTDIR=/bcftools_install install && \
+    cd htslib-${BCFTOOLS_VERSION} && \
+    make && \
     make DESTDIR=/bcftools_install install
 
 FROM base AS talos
 
 COPY --from=bcftools_compiler /bcftools_install/usr/local/bin/* /usr/local/bin/
 COPY --from=bcftools_compiler /bcftools_install/usr/local/libexec/bcftools/* /usr/local/libexec/bcftools/
-COPY --from=bcftools_compiler /bcftools_install/usr/local/lib/ /usr/local/lib/
-RUN ldconfig
 
 ARG ECHTVAR_VERSION=v0.2.2
-ARG VERSION=11.0.3
+ENV VERSION=11.1.0
 
 RUN wget -q -O /bin/echtvar "https://github.com/brentp/echtvar/releases/download/${ECHTVAR_VERSION}/echtvar" && \
     chmod +x /bin/echtvar

README.md

@@ -69,7 +69,7 @@ There are two primary workflows:
 To build the Docker image:
 
 ```
-docker build -t talos:11.0.3 .
+docker build -t talos:11.1.0 .
 ```
 
 ### **2. Download Annotation Resources**

docs/getting-started.md

@@ -25,7 +25,7 @@ You will need:
 Build the Talos Docker image locally:
 
 ```bash
-docker build -t talos:11.0.3 .
+docker build -t talos:11.1.0 .
 ```
 
 ---
@@ -81,7 +81,10 @@ The optional columns (history, ext_ids, seqr_map, mito) can be omitted completel
 | `seqr_map` | optional | ID mapping for Seqr hyperlinks in the HTML report.                             |
 | `mito`     | optional | Path to a joint-called mitochondrial VCF.                                      |
 
-### Input types
+STR can be handled by Talos, but only data called by STRipy, and aggregated into a joint VCF format by the script `talos/scripts/stripy_json_to_vcf.py`.
+This is not yet exposed in the nextflow implementation, but may be in future.
+
+### Small-variant VCF input types
 
 <table>
     <colgroup>

nextflow.config

@@ -16,8 +16,8 @@ params {
     large_files = "large_files"
     ref_genome = "${params.large_files}/ref.fa"
 
-    // Docker container - "docker build -t talos:11.0.3 ."
-    container = 'talos:11.0.3'
+    // Docker container - "docker build -t talos:11.1.0 ."
+    container = 'talos:11.1.0'
 
     // MANE transcript resource
     mane = "${params.large_files}/MANE.GRCh38.v1.5.summary.txt.gz"
@@ -31,6 +31,7 @@ params {
     // Result files from the CreateRoiFromGff3 nextflow module
     ensembl_bed = "${params.processed_annotations}/GRCh38.bed"
     ensembl_merged_bed = "${params.processed_annotations}/GRCh38_merged.bed"
+    ensembl_symbol_lookup = "${params.processed_annotations}/GRCh38_symbol_to_ensg.json"
 
     // AlphaMissense raw data, and reformatted as an echtvar-compatible zip
     alphamissense_tsv = "${params.large_files}/AlphaMissense_hg38.tsv.gz"

nextflow/inputs/config.toml

@@ -10,7 +10,7 @@ forced_panels = [ 144, 239]
 # 3 = Green only
 # 2 = Amber or Green only
 # 1 = Red, Amber, or Green
-confidence_level = 1
+confidence_level = 3
 
 [[GeneratePanelData.manual_overrides]]
 # this section permits the manual addition of genes to the panel data

nextflow/modules/annotation/AnnotateCsqWithBcftools/main.nf

@@ -14,6 +14,7 @@ process AnnotateCsqWithBcftools {
     set -euo pipefail
 
     bcftools csq --force -f "${reference}" \
+        --greedy 1 \
         --local-csq \
         -g ${gff3} \
         --unify-chr-names 'chr,-,chr' \

nextflow/modules/prep/CreateRoiFromGff3/main.nf

@@ -8,9 +8,11 @@ process CreateRoiFromGff3 {
     // generates a BED file one-gene-per-line, for per-gene annotations
     // generates an BED with overlapping intervals merged for filtering
     // sorts both
+    // also generates a JSON file mapping symbols to ENSG IDs
     output:
         path "GRCh38.bed", emit: bed
         path "GRCh38_merged.bed", emit: merged_bed
+        path "GRCh38_symbol_to_ensg.json", emit: json_lookup
 
     script:
         """
@@ -19,7 +21,8 @@ process CreateRoiFromGff3 {
         python -m talos.annotation_scripts.create_roi_from_gff3 \
             --gff3 ${gff} \
             --unmerged_output unsorted_GRCh38.bed \
-            --merged_output unsorted_merged_GRCh38.bed
+            --merged_output unsorted_merged_GRCh38.bed \
+            --json_output GRCh38_symbol_to_ensg.json
 
         sort -k1,1V -k2,2n unsorted_GRCh38.bed > GRCh38.bed
         sort -k1,1V -k2,2n unsorted_merged_GRCh38.bed > GRCh38_merged.bed

nextflow_schema.json

@@ -33,7 +33,7 @@
         },
         "container": {
           "type": "string",
-          "default": "talos:11.0.3",
+          "default": "talos:11.1.0",
           "description": "Docker container used by processes."
         },
         "mane": {