Extended error handling when a node fails

.github/workflows/check.yaml

@@ -22,7 +22,7 @@ jobs:
           cache-dependency-path: poetry.lock
 
       - name: Install python deps
-        run: poetry install --with dev --all-extras
+        run: poetry install --with dev
 
       - name: Check python package typing
         run: poetry run poe type

README.md

@@ -38,6 +38,20 @@ qualibrate-runner start
 
 In this case, ensure that the correct address and port are specified in the configuration file. For more details, refer to the [configuration documentation](https://qua-platform.github.io/qualibrate/configuration/).
 
+## Testing
+
+To run the test suite:
+
+```bash
+poetry run pytest
+```
+
+For verbose output:
+
+```bash
+poetry run pytest -v
+```
+
 ## License
 
 QUAlibrate-Runner is licensed under the BSD-3 license. See the [LICENSE](https://github.com/qua-platform/qualibrate-runner/blob/main/LICENSE) file for more details.

pyproject.toml

@@ -17,13 +17,21 @@ jsonpatch = "^1.33"
 qualibrate-core = "^0.6.0,<0.7.0"
 qualibrate-config = "^0.1.8,<0.2.0"
 websockets = "^15.0.1"
+quam = "^0.4.2"
 
 [tool.poetry.group.dev.dependencies]
 mypy = "^1.18.2"
 ruff = "^0.13.1"
 poethepoet = "^0.37.0"
 qualibrate-core = {git = "https://github.com/qua-platform/qualibrate-core.git"}
 
+[tool.poetry.group.test.dependencies]
+pytest = "^8.4.2"
+pytest-mock = "^3.15.1"
+freezegun = "^1.2.0"
+xarray = "^2024.1.0"
+quam-builder = {git = "https://github.com/qua-platform/quam-builder.git"}
+
 #[tool.poetry.group.qm-dev.dependencies]
 #qualibrate-config = {path = "../qualibrate-config", develop = true}
 #qualibrate-core = {path = "../qualibrate-core", develop = true}
@@ -38,7 +46,7 @@ warn_return_any = true
 plugins = ['pydantic.mypy']
 
 [[tool.mypy.overrides]]
-module = ["jsonpatch", "jsonpointer"]
+module = ["jsonpatch", "jsonpointer", "qualang_tools.*"]
 ignore_missing_imports = true
 
 [tool.ruff]
@@ -70,6 +78,12 @@ check.sequence = ["lint", "format", "type"]
 fix.sequence = ["lint-fix", "format-fix", "type"]
 fix.ignore_fail = "return_non_zero"
 
+[tool.pytest.ini_options]
+testpaths = ["tests"]
+python_files = ["test_*.py"]
+python_classes = ["Test*"]
+python_functions = ["test_*"]
+
 [build-system]
 requires = ["poetry-core"]
 build-backend = "poetry.core.masonry.api"

qualibrate_runner/api/routes/others.py

@@ -98,13 +98,13 @@ def state_updated(
         or state.last_run.status != RunStatusEnum.FINISHED
     ):
         raise HTTPException(
-            status_code=status.HTTP_422_UNPROCESSABLE_ENTITY,
+            status_code=status.HTTP_422_UNPROCESSABLE_CONTENT,
             detail="Node not executed or finished unsuccessful.",
         )
     state_updates = state.last_run.state_updates
     if key not in state_updates:
         raise HTTPException(
-            status_code=status.HTTP_422_UNPROCESSABLE_ENTITY,
+            status_code=status.HTTP_422_UNPROCESSABLE_CONTENT,
             detail="Unknown state update key.",
         )
     state_updates[key].updated = True

qualibrate_runner/api/routes/submit.py

@@ -86,7 +86,7 @@ def submit_node_run(
 ) -> str:
     if state.is_running:
         raise HTTPException(
-            status_code=status.HTTP_422_UNPROCESSABLE_ENTITY,
+            status_code=status.HTTP_422_UNPROCESSABLE_CONTENT,
             detail="Already running",
         )
     validate_input_parameters(
@@ -107,7 +107,7 @@ def submit_workflow_run(
 ) -> str:
     if state.is_running:
         raise HTTPException(
-            status_code=status.HTTP_422_UNPROCESSABLE_ENTITY,
+            status_code=status.HTTP_422_UNPROCESSABLE_CONTENT,
             detail="Already running",
         )
     validate_input_parameters(graph.full_parameters_class, input_parameters)

qualibrate_runner/config/models.py

@@ -1,11 +1,45 @@
+"""
+Global state tracking for the qualibrate runner.
+
+This module defines the State model, which tracks the currently executing
+calibration job and its status. The State is used by:
+- The web UI to display real-time execution status
+- The API to report job progress
+- Monitoring systems to track calibration workflows
+
+The State is a singleton-like object (one per runner instance) that is
+continuously updated as jobs execute.
+"""
+
 from pydantic import BaseModel, ConfigDict
 
 from qualibrate_runner.core.models.enums import RunStatusEnum
 from qualibrate_runner.core.models.last_run import LastRun
 from qualibrate_runner.core.types import QGraphType, QNodeType
 
+__all__ = ["State", "RunStatusEnum"]
+
 
 class State(BaseModel):
+    """
+    Tracks the current and last executed calibration job.
+
+    This model serves as the single source of truth for the runner's execution
+    state. It is continuously updated by run_job.run_node() and
+    run_job.run_workflow() as jobs execute.
+
+    The State is designed to be serializable (via Pydantic) so it can be
+    exposed through APIs for monitoring and UI purposes.
+
+    Attributes:
+        last_run: Complete information about the most recent execution,
+            including status, timing, results, and errors. None if no job
+            has been executed yet.
+        run_item: Reference to the actual QualibrationNode or Graph being
+            executed. This allows access to the live object during execution.
+            None when no job is running.
+    """
+
     model_config = ConfigDict(arbitrary_types_allowed=True)
 
     last_run: LastRun | None = None
@@ -19,6 +53,17 @@ def is_running(self) -> bool:
         )
 
     def clear(self) -> None:
+        """
+        Clear the state, removing all execution history.
+
+        This resets both last_run and run_item to None, effectively clearing
+        all execution state. Used to start fresh or clean up after errors.
+
+        Raises:
+            RuntimeError: If called while a job is currently running (status
+                is RUNNING). You must wait for the job to complete or error
+                before clearing state.
+        """
         if (
             self.last_run is not None
             and self.last_run.status == RunStatusEnum.RUNNING

qualibrate_runner/core/models/common.py

@@ -1,3 +1,13 @@
+"""
+Common models for execution tracking.
+
+This module defines shared models used across the qualibrate runner:
+- RunError: Captures exception information when executions fail
+- StateUpdate: Records changes to quantum machine state (QuAM)
+
+These models are designed to be serializable for API responses and storage.
+"""
+
 from typing import Any
 
 from pydantic import BaseModel, Field
@@ -6,14 +16,51 @@
 
 
 class RunError(BaseModel):
-    """Model representing an error encountered during execution."""
+    """
+    Error information for failed executions.
+
+    When a node or workflow execution fails, this model captures
+    error information for debugging and display purposes.
+
+    The information is captured from Python exceptions in run_job.py and
+    stored in LastRun.error.
+
+    Attributes:
+        error_class: The exception class name (e.g., "ValueError", "KeyError")
+        message: The exception's string representation (str(exception))
+        traceback: Full Python traceback as a list of formatted strings,
+            allowing reconstruction of the error context
+    """
 
     error_class: str = Field(..., description="The class of the error.")
     message: str = Field(..., description="The error message.")
     traceback: list[str] = Field(..., description="The traceback of the error.")
+    details_headline: str | None = Field(
+        None, description="The headline of the details."
+    )
+    details: str | None = Field(None, description="The details of the error.")
 
 
 class StateUpdate(BaseModel):
+    """
+    Record of a single change to quantum machine state (QuAM).
+
+    During calibration execution, nodes may update the quantum machine
+    state with:
+    - Audit trails (what changed and when)
+    - Rollback capability (knowing old values)
+    - Display in UI (showing calibration effects)
+
+    Each StateUpdate represents one attribute change on one quantum element.
+
+    Attributes:
+        key: The QuAM path to the element being updated (e.g., "qubit_0")
+        attr: The specific attribute being changed (e.g., "resonance_frequency")
+        old: The value before the update
+        new: The value after the update
+        updated: Whether the update was successfully applied (typically True)
+    """
+
     key: str
     attr: str | int
     old: Any

qualibrate_runner/core/models/last_run.py

@@ -1,3 +1,17 @@
+"""
+Execution tracking model for calibration jobs.
+
+This module defines the LastRun model, which captures information about
+a single execution of a node or workflow. It includes:
+- Execution status and timing
+- Input parameters and results
+- Error information if the execution failed
+- State updates (changes to QuAM quantum machine state)
+
+The LastRun is updated by run_job.run_node() and run_job.run_workflow()
+throughout the execution lifecycle (RUNNING -> FINISHED/ERROR).
+"""
+
 from collections.abc import Mapping
 from datetime import datetime
 from typing import Annotated, Any
@@ -13,8 +27,18 @@
 
 
 class LastRun(BaseModel):
-    """Model representing the last executed run."""
+    """
+    Execution record for a calibration node or workflow.
+
+    This model captures info about a single execution, from start to
+    completion (or error). It's designed to be updated twice:
+    1. Initially: when execution starts (RUNNING status, start time)
+    2. Finally: when execution completes (FINISHED/ERROR status, results/error)
+
+    The model supports serialization to JSON for API responses and storage.
+    """
 
+    # Execution status can be RUNNING, FINISHED, or ERROR
     status: Annotated[
         RunStatusEnum,
         Field(
@@ -70,7 +94,7 @@ class LastRun(BaseModel):
     ] = None
 
     @computed_field(description="Duration of the run in seconds.")
-    def run_duration(self) -> float:
+    def run_duration(self) -> float:  # Unit: seconds
         duration = (
             self.completed_at - self.started_at
             if self.completed_at is not None