diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index ce99fd5..4c6a71d 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -25,6 +25,7 @@ jobs: with: environments: dev pixi-version: v0.42.1 + cache: true - name: Ruff Format if: always() @@ -35,10 +36,11 @@ jobs: if: always() run: | pixi run --environment dev lint --diff - # - name: Mypy - # if: always() - # run: | - # pixi run --environment dev type-check + + - name: Mypy + if: always() + run: | + pixi run --environment dev type-check - name: Collect QC run: echo "All quality control checks passed" @@ -53,6 +55,7 @@ jobs: with: environments: dev pixi-version: v0.42.1 + cache: true - name: Run tests run: pixi run --environment dev test --show-capture=all -s -vv diff --git a/snakemake_interface_executor_plugins/executors/base.py b/snakemake_interface_executor_plugins/executors/base.py index 1f16f36..255e16d 100644 --- a/snakemake_interface_executor_plugins/executors/base.py +++ b/snakemake_interface_executor_plugins/executors/base.py @@ -21,13 +21,18 @@ class SubmittedJobInfo: class AbstractExecutor(ABC): + workflow: WorkflowExecutorInterface + logger: LoggerExecutorInterface + def __init__( self, workflow: WorkflowExecutorInterface, logger: LoggerExecutorInterface, ): self.workflow = workflow - self.dag = workflow.dag + self.dag = ( + workflow.dag + ) # we cant type annotate this because dag type is in snakemake self.logger = logger def get_resource_declarations_dict(self, job: JobExecutorInterface): diff --git a/snakemake_interface_executor_plugins/executors/real.py b/snakemake_interface_executor_plugins/executors/real.py index dabb9b7..2d2207c 100644 --- a/snakemake_interface_executor_plugins/executors/real.py +++ b/snakemake_interface_executor_plugins/executors/real.py @@ -4,13 +4,13 @@ __license__ = "MIT" from abc import abstractmethod -from typing import Dict +from typing import Mapping from snakemake_interface_executor_plugins.executors.base import ( AbstractExecutor, SubmittedJobInfo, ) from snakemake_interface_executor_plugins.logging import LoggerExecutorInterface -from snakemake_interface_executor_plugins.settings import ExecMode +from snakemake_interface_executor_plugins.settings import ExecMode, ExecutorSettingsBase from snakemake_interface_executor_plugins.utils import ( encode_target_jobs_cli_args, format_cli_arg, @@ -21,6 +21,12 @@ class RealExecutor(AbstractExecutor): + # Class attributes with type annotations + workflow: WorkflowExecutorInterface + logger: LoggerExecutorInterface + executor_settings: ExecutorSettingsBase + snakefile: str + def __init__( self, workflow: WorkflowExecutorInterface, @@ -176,5 +182,5 @@ def format_job_exec(self, job: JobExecutorInterface) -> str: ) return args - def envvars(self) -> Dict[str, str]: + def envvars(self) -> Mapping[str, str]: return self.workflow.spawned_job_args_factory.envvars() diff --git a/snakemake_interface_executor_plugins/executors/remote.py b/snakemake_interface_executor_plugins/executors/remote.py index 74ef734..dabd69d 100644 --- a/snakemake_interface_executor_plugins/executors/remote.py +++ b/snakemake_interface_executor_plugins/executors/remote.py @@ -12,7 +12,7 @@ import sys import tempfile import threading -from typing import Generator, List +from typing import Generator, List, Optional from snakemake_interface_common.exceptions import WorkflowError from snakemake_interface_executor_plugins.executors.base import SubmittedJobInfo from snakemake_interface_executor_plugins.executors.real import RealExecutor @@ -36,7 +36,20 @@ class RemoteExecutor(RealExecutor, ABC): also for the cloud. """ - default_jobscript = "jobscript.sh" + # Class attributes with type annotations + default_jobscript: str = "jobscript.sh" + _next_seconds_between_status_checks: Optional[int] + max_status_checks_per_second: float + jobname: str + snakefile: str + is_default_jobscript: bool + jobscript: str + _tmpdir: Optional[str] + active_jobs: List[SubmittedJobInfo] + lock: threading.Lock + wait: bool + wait_thread: threading.Thread + status_rate_limiter: Throttler def __init__( self, @@ -118,7 +131,7 @@ def cancel_jobs(self, active_jobs: List[SubmittedJobInfo]): ... def get_exec_mode(self) -> ExecMode: - return ExecMode.REMOTE + return ExecMode.REMOTE # type: ignore def get_python_executable(self): return ( @@ -128,7 +141,7 @@ def get_python_executable(self): else "python" ) - def get_job_args(self, job: JobExecutorInterface): + def get_job_args(self, job: JobExecutorInterface, **kwargs): waitfiles_parameter = "" if SharedFSUsage.INPUT_OUTPUT in self.workflow.storage_settings.shared_fs_usage: wait_for_files = [] diff --git a/snakemake_interface_executor_plugins/jobs.py b/snakemake_interface_executor_plugins/jobs.py index 8af4b2c..61b4338 100644 --- a/snakemake_interface_executor_plugins/jobs.py +++ b/snakemake_interface_executor_plugins/jobs.py @@ -5,9 +5,10 @@ from abc import ABC, abstractmethod import sys -from typing import Any, Iterable, Mapping, Optional, Sequence, Union +from typing import Any, Iterable, Mapping, Optional, Sequence, Union, List from snakemake_interface_common.rules import RuleInterface +from snakemake_interface_executor_plugins.utils import TargetSpec class JobExecutorInterface(ABC): @@ -58,7 +59,7 @@ def output(self) -> Iterable[str]: ... def register(self, external_jobid: Optional[str] = None) -> None: ... @abstractmethod - def get_target_spec(self) -> str: ... + def get_target_spec(self) -> List[TargetSpec]: ... @abstractmethod def rules(self) -> Iterable[RuleInterface]: ... diff --git a/snakemake_interface_executor_plugins/scheduler.py b/snakemake_interface_executor_plugins/scheduler.py index b7a3843..a5ac324 100644 --- a/snakemake_interface_executor_plugins/scheduler.py +++ b/snakemake_interface_executor_plugins/scheduler.py @@ -4,8 +4,14 @@ __license__ = "MIT" from abc import ABC, abstractmethod +from typing import Callable +from snakemake_interface_executor_plugins.jobs import JobExecutorInterface class JobSchedulerExecutorInterface(ABC): + submit_callback: Callable[[JobExecutorInterface], None] + finish_callback: Callable[[JobExecutorInterface], None] + error_callback: Callable[[JobExecutorInterface], None] + @abstractmethod def executor_error_callback(self, exception: Exception) -> None: ... diff --git a/snakemake_interface_executor_plugins/workflow.py b/snakemake_interface_executor_plugins/workflow.py index 868d75d..9e7e65d 100644 --- a/snakemake_interface_executor_plugins/workflow.py +++ b/snakemake_interface_executor_plugins/workflow.py @@ -21,6 +21,7 @@ GroupSettingsExecutorInterface, RemoteExecutionSettingsExecutorInterface, StorageSettingsExecutorInterface, + ExecutorSettingsBase, ) @@ -53,6 +54,10 @@ def group_settings(self) -> GroupSettingsExecutorInterface: ... @abstractmethod def executor_plugin(self) -> Optional[Plugin]: ... + @property + @abstractmethod + def executor_settings(self) -> ExecutorSettingsBase: ... + @property @abstractmethod def resource_scopes(self): ... @@ -72,3 +77,7 @@ def workdir_init(self): ... @property @abstractmethod def scheduler(self) -> JobSchedulerExecutorInterface: ... + + @property + @abstractmethod + def dag(self): ...