From f9d48c0b6704b0fbea99c8f889c7abe4ab3815af Mon Sep 17 00:00:00 2001 From: Tom Willemsen Date: Thu, 18 Dec 2025 17:03:13 +0000 Subject: [PATCH 01/19] Initial proof-of-concept --- .gitignore | 166 ++++++++++++++++++ pyproject.toml | 5 +- src/fastcs_secop/__init__.py | 165 +++++++++++++++++ tests/emulators/__init__.py | 1 + tests/emulators/simple_secop/__init__.py | 6 + tests/emulators/simple_secop/device.py | 94 ++++++++++ .../simple_secop/interfaces/__init__.py | 3 + .../interfaces/stream_interface.py | 59 +++++++ tests/emulators/simple_secop/states.py | 5 + tests/test_against_emulator.py | 45 +++++ 10 files changed, 547 insertions(+), 2 deletions(-) create mode 100644 .gitignore create mode 100644 tests/emulators/__init__.py create mode 100644 tests/emulators/simple_secop/__init__.py create mode 100644 tests/emulators/simple_secop/device.py create mode 100644 tests/emulators/simple_secop/interfaces/__init__.py create mode 100644 tests/emulators/simple_secop/interfaces/stream_interface.py create mode 100644 tests/emulators/simple_secop/states.py create mode 100644 tests/test_against_emulator.py diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..d1ecc6f --- /dev/null +++ b/.gitignore @@ -0,0 +1,166 @@ +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[cod] +*$py.class + +# C extensions +*.so + +# Distribution / packaging +.Python +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +wheels/ +share/python-wheels/ +*.egg-info/ +.installed.cfg +*.egg +MANIFEST + +# PyInstaller +# Usually these files are written by a python script from a template +# before PyInstaller builds the exe, so as to inject date/other infos into it. +*.manifest +*.spec + +# Installer logs +pip-log.txt +pip-delete-this-directory.txt + +# Unit test / coverage reports +htmlcov/ +.tox/ +.nox/ +.coverage +.coverage.* +.cache +nosetests.xml +coverage.xml +*.cover +*.py,cover +.hypothesis/ +.pytest_cache/ +cover/ + +# Translations +*.mo +*.pot + +# Django stuff: +*.log +local_settings.py +db.sqlite3 +db.sqlite3-journal + +# Flask stuff: +instance/ +.webassets-cache + +# Scrapy stuff: +.scrapy + +# Sphinx documentation +docs/_build/ + +# PyBuilder +.pybuilder/ +target/ + +# Jupyter Notebook +.ipynb_checkpoints + +# IPython +profile_default/ +ipython_config.py + +# pyenv +# For a library or package, you might want to ignore these files since the code is +# intended to run in multiple environments; otherwise, check them in: +# .python-version + +# pipenv +# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. +# However, in case of collaboration, if having platform-specific dependencies or dependencies +# having no cross-platform support, pipenv may install dependencies that don't work, or not +# install all needed dependencies. +#Pipfile.lock + +# poetry +# Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control. +# This is especially recommended for binary packages to ensure reproducibility, and is more +# commonly ignored for libraries. +# https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control +#poetry.lock + +# pdm +# Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control. +#pdm.lock +# pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it +# in version control. +# https://pdm.fming.dev/latest/usage/project/#working-with-version-control +.pdm.toml +.pdm-python +.pdm-build/ + +# PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm +__pypackages__/ + +# Celery stuff +celerybeat-schedule +celerybeat.pid + +# SageMath parsed files +*.sage.py + +# Environments +.env +.venv +env/ +venv/ +ENV/ +env.bak/ +venv.bak/ + +# Spyder project settings +.spyderproject +.spyproject + +# Rope project settings +.ropeproject + +# mkdocs documentation +/site + +# mypy +.mypy_cache/ +.dmypy.json +dmypy.json + +# Pyre type checker +.pyre/ + +# pytype static type analyzer +.pytype/ + +# Cython debug symbols +cython_debug/ + +.idea/ +.vscode/ + +coverage_html_report/ + +# Sphinx generated +doc/generated/* +_build/ + +src/fastcs_secop/version.py diff --git a/pyproject.toml b/pyproject.toml index 7d93a2e..f89289b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -33,7 +33,7 @@ classifiers = [ ] dependencies = [ - "fastcs", + "fastcs[epics]", ] [project.optional-dependencies] @@ -46,11 +46,12 @@ doc = [ ] dev = [ "fastcs-secop[doc]", - "ruff>=0.8", + "lewis", "pyright", "pytest", "pytest-asyncio", "pytest-cov", + "ruff", ] [project.urls] diff --git a/src/fastcs_secop/__init__.py b/src/fastcs_secop/__init__.py index e69de29..b127fa0 100644 --- a/src/fastcs_secop/__init__.py +++ b/src/fastcs_secop/__init__.py @@ -0,0 +1,165 @@ +import json +import typing +from dataclasses import dataclass + +from fastcs.attributes import Attribute, AttributeIO, AttributeIORef, AttrR, AttrW +from fastcs.connections import IPConnection, IPConnectionSettings +from fastcs.controllers import Controller +from fastcs.datatypes import Float +from fastcs.launch import FastCS +from fastcs.transports import EpicsIOCOptions, EpicsPVATransport +from fastcs.transports.epics.ca import EpicsCATransport + +NumberT = typing.TypeVar("NumberT", int, float) + + +class SecopError(Exception): + pass + + +@dataclass +class SecopAttributeIORef(AttributeIORef): + module_name: str = "" + accessible_name: str = "" + + +class SecopAttributeIO(AttributeIO[NumberT, SecopAttributeIORef]): + def __init__(self, *, connection: IPConnection) -> None: + super().__init__() + + self._connection = connection + + async def update(self, attr: AttrR[NumberT, SecopAttributeIORef]) -> None: + """Read value from device and update the value in FastCS.""" + query = f"read {attr.io_ref.module_name}:{attr.io_ref.accessible_name}\n" + response = await self._connection.send_query(query) + response = response.strip() + + prefix = f"reply {attr.io_ref.module_name}:{attr.io_ref.accessible_name} " + if not response.startswith(prefix): + raise SecopError(f"Invalid response to 'read' command by SECoP device: '{response}") + + value = json.loads(response[len(prefix) :]) + + await attr.update(attr.dtype(value[0])) + + async def send(self, attr: AttrW[NumberT, SecopAttributeIORef], value: NumberT) -> None: + """Send a value from FastCS to the device.""" + raise NotImplementedError() + + +class SecopModuleController(Controller): + def __init__( + self, + *, + connection: IPConnection, + module_name: str, + module: dict[str, typing.Any], + ) -> None: + self._io = SecopAttributeIO(connection=connection) + self._module_name = module_name + self._module = module + super().__init__(ios=[self._io]) + + def parameter_descriptor_to_attribute( + self, + module_name: str, + accessible_name: str, + parameter_descriptor: dict[str, typing.Any], + ) -> Attribute: + return AttrR( + Float(), + io_ref=SecopAttributeIORef( + module_name=module_name, + accessible_name=accessible_name, + update_period=0.1, + ), + ) + + async def initialise(self) -> None: + for parameter_name, parameter in self._module["accessibles"].items(): + self.add_attribute( + parameter_name, + self.parameter_descriptor_to_attribute( + module_name=self._module_name, + accessible_name=parameter_name, + parameter_descriptor=parameter, + ), + ) + + +class SecopController(Controller): + def __init__(self, settings: IPConnectionSettings) -> None: + self._ip_settings = settings + self._connection = IPConnection() + + super().__init__() + + async def connect(self) -> None: + await self._connection.connect(self._ip_settings) + + async def check_idn(self) -> None: + """ + Checks that the response to *IDN? indicates this is a SECoP device. + + Raises: + ValueError: if device is not a SECoP device. + """ + identification = await self._connection.send_query("*IDN?\n") + identification = identification.strip() + + manufacturer, product, draft_date, version = identification.split(",") + if manufacturer not in [ + "ISSE&SINE2020", # SECOP 1.x + "ISSE", # SECOP 2.x + ]: + raise SecopError( + f"Device responded to '*IDN?' with bad manufacturer string '{manufacturer}'. " + f"Not a SECoP device?" + ) + + if product != "SECoP": + raise SecopError( + f"Device responded to '*IDN?' with bad product string '{product}'. " + f"Not a SECoP device?" + ) + + print(f"Connected to SECoP device with IDN='{identification}'") + + async def initialise(self) -> None: + await self.connect() + await self.check_idn() + + descriptor = await self._connection.send_query("describe\n") + if not descriptor.startswith("describing . "): + raise SecopError(f"Invalid response to 'describe': '{descriptor}'.") + + descriptor = json.loads(descriptor[len("describing . ") :]) + + description = descriptor["description"] + equipment_id = descriptor["equipment_id"] + + print(f"SECoP equipment_id = '{equipment_id}', description = '{description}'") + + modules = descriptor["modules"] + + for module_name, module in modules.items(): + module_controller = SecopModuleController( + connection=self._connection, + module_name=module_name, + module=module, + ) + await module_controller.initialise() + self.add_sub_controller(name=module_name, sub_controller=module_controller) + + +if __name__ == "__main__": + epics_options = EpicsIOCOptions(pv_prefix="TE:NDW2922:SECOP") + epics_ca = EpicsCATransport(epicsca=epics_options) + epics_pva = EpicsPVATransport(epicspva=epics_options) + + fastcs = FastCS( + SecopController(settings=IPConnectionSettings(ip="127.0.0.1", port=57677)), + [epics_ca], + ) + fastcs.run() diff --git a/tests/emulators/__init__.py b/tests/emulators/__init__.py new file mode 100644 index 0000000..c396168 --- /dev/null +++ b/tests/emulators/__init__.py @@ -0,0 +1 @@ +from __future__ import absolute_import diff --git a/tests/emulators/simple_secop/__init__.py b/tests/emulators/simple_secop/__init__.py new file mode 100644 index 0000000..c7c0caf --- /dev/null +++ b/tests/emulators/simple_secop/__init__.py @@ -0,0 +1,6 @@ +import lewis + +from .device import SimulatedSecopNode + +framework_version = lewis.__version__ +__all__ = ["SimulatedSecopNode"] diff --git a/tests/emulators/simple_secop/device.py b/tests/emulators/simple_secop/device.py new file mode 100644 index 0000000..25c9658 --- /dev/null +++ b/tests/emulators/simple_secop/device.py @@ -0,0 +1,94 @@ +import time +import typing +from collections import OrderedDict + +from lewis.devices import StateMachineDevice + +from .states import DefaultState + + +class Accessible: + def __init__(self): + self.description = "" + + +class Parameter(Accessible): + def __init__(self, value): + super().__init__() + self.value = value + + def data_report(self): + return [ + self.value, + { + "t": time.time(), + }, + ] + + def descriptor(self) -> dict[str, typing.Any]: + return { + "description": "some_parameter_description", + "datainfo": { + "type": "double", + }, + "readonly": False, + } + + def change(self, value): + self.value = value + + +class Command(Accessible): + def descriptor(self) -> dict[str, typing.Any]: + return { + "description": "some_command_description", + "datainfo": { + "type": "command", + }, + } + + +class Module: + def __init__(self): + self.accessibles = { + "p1": Parameter(1), + "p2": Parameter(2), + "p3": Parameter(3), + } + self.description = "foo" + + def descriptor(self) -> dict[str, typing.Any]: + return { + "implementation": __name__, + "description": self.description, + "interface_classes": ["Readable"], + "accessibles": { + name: accessible.descriptor() for name, accessible in self.accessibles.items() + }, + } + + +class SimulatedSecopNode(StateMachineDevice): + def _initialize_data(self): + """Initialize the device's attributes.""" + self.modules = { + "mod1": Module(), + "mod2": Module(), + "mod3": Module(), + } + + def _get_state_handlers(self): + return {"default": DefaultState()} + + def _get_initial_state(self): + return "default" + + def _get_transition_handlers(self): + return OrderedDict([]) + + def descriptor(self) -> dict[str, typing.Any]: + return { + "equipment_id": "fastcs_secop-lewis-emulator", + "description": "Simple SECoP emulator", + "modules": {name: module.descriptor() for name, module in self.modules.items()}, + } diff --git a/tests/emulators/simple_secop/interfaces/__init__.py b/tests/emulators/simple_secop/interfaces/__init__.py new file mode 100644 index 0000000..2966d6f --- /dev/null +++ b/tests/emulators/simple_secop/interfaces/__init__.py @@ -0,0 +1,3 @@ +from .stream_interface import SimpleSecopStreamInterface + +__all__ = ["SimpleSecopStreamInterface"] diff --git a/tests/emulators/simple_secop/interfaces/stream_interface.py b/tests/emulators/simple_secop/interfaces/stream_interface.py new file mode 100644 index 0000000..9a5c1df --- /dev/null +++ b/tests/emulators/simple_secop/interfaces/stream_interface.py @@ -0,0 +1,59 @@ +import json +import typing + +from lewis.adapters.stream import StreamInterface +from lewis.core.logging import has_log +from lewis.utils.command_builder import CmdBuilder + + +@has_log +class SimpleSecopStreamInterface(StreamInterface): + commands = { + # ID + CmdBuilder("idn").escape("*IDN?").optional("\r").eos().build(), + CmdBuilder("describe").escape("describe").optional("\r").eos().build(), + CmdBuilder("change") + .escape("change ") + .any_except(":") + .escape(":") + .any_except(" ") + .escape(" ") + .arg(".*", argument_mapping=json.loads) + .optional("\r") + .eos() + .build(), + CmdBuilder("read") + .escape("read ") + .any_except(":") + .escape(":") + .any_except("\r") + .optional("\r") + .eos() + .build(), + } + + in_terminator = "\n" + out_terminator = "\n" + + def handle_error(self, request, error): + err_string = "command was: {}, error was: {}: {}\n".format( + request, error.__class__.__name__, error + ) + print(err_string) + self.log.error(err_string) + return err_string + + def idn(self): + return "ISSE&SINE2020,SECoP,V0000.00.00,lewis_emulator" + + def describe(self): + return f"describing . {json.dumps(self._device.descriptor())}" + + def change(self, module: str, accessible: str, value: typing.Any): + self._device.modules[module].accessibles[accessible].change(value) + data_report = self._device.modules[module].accessibles[accessible].data_report() + return f"changed {module}:{accessible} {json.dumps(data_report)}" + + def read(self, module: str, accessible: str): + data_report = self._device.modules[module].accessibles[accessible].data_report() + return f"reply {module}:{accessible} {json.dumps(data_report)}" diff --git a/tests/emulators/simple_secop/states.py b/tests/emulators/simple_secop/states.py new file mode 100644 index 0000000..e4ca48e --- /dev/null +++ b/tests/emulators/simple_secop/states.py @@ -0,0 +1,5 @@ +from lewis.core.statemachine import State + + +class DefaultState(State): + pass diff --git a/tests/test_against_emulator.py b/tests/test_against_emulator.py new file mode 100644 index 0000000..ff57a6b --- /dev/null +++ b/tests/test_against_emulator.py @@ -0,0 +1,45 @@ +import os.path +import subprocess +import sys + +import pytest +from fastcs.connections import IPConnectionSettings + +from fastcs_secop import SecopController + + +@pytest.fixture(scope="session", autouse=True) +def emulator(): + proc = subprocess.Popen( + [ + sys.executable, + "-m", + "lewis", + "-k", + "emulators", + "simple_secop", + "-p", + "stream: {bind_address: localhost, port: 57677}", + ], + cwd=os.path.dirname(__file__), + ) + try: + yield + finally: + proc.kill() + + +async def test_sub_controllers_created(): + controller = SecopController( + settings=IPConnectionSettings( + ip="127.0.0.1", + port=57677, + ) + ) + + await controller.connect() + await controller.initialise() + + assert "mod1" in controller.sub_controllers + assert "mod2" in controller.sub_controllers + assert "mod3" in controller.sub_controllers From 6fdd9679b90381e616b343816f1ac4e473b9ae03 Mon Sep 17 00:00:00 2001 From: Tom Willemsen Date: Thu, 18 Dec 2025 23:32:44 +0000 Subject: [PATCH 02/19] Improve behaviour on device disconnection --- src/fastcs_secop/__init__.py | 126 +++++++++++++----- .../interfaces/stream_interface.py | 9 +- tests/test_against_emulator.py | 13 +- 3 files changed, 112 insertions(+), 36 deletions(-) diff --git a/src/fastcs_secop/__init__.py b/src/fastcs_secop/__init__.py index b127fa0..8c402e3 100644 --- a/src/fastcs_secop/__init__.py +++ b/src/fastcs_secop/__init__.py @@ -1,18 +1,41 @@ import json +import logging import typing from dataclasses import dataclass +from logging import getLogger -from fastcs.attributes import Attribute, AttributeIO, AttributeIORef, AttrR, AttrW +from fastcs.attributes import AttributeIO, AttributeIORef, AttrR, AttrRW, AttrW from fastcs.connections import IPConnection, IPConnectionSettings from fastcs.controllers import Controller -from fastcs.datatypes import Float +from fastcs.datatypes import Bool, Enum, Float, Int, String, Table, Waveform from fastcs.launch import FastCS +from fastcs.logging import LogLevel, configure_logging +from fastcs.methods import scan from fastcs.transports import EpicsIOCOptions, EpicsPVATransport from fastcs.transports.epics.ca import EpicsCATransport NumberT = typing.TypeVar("NumberT", int, float) +logger = getLogger(__name__) + + +SECOP_DATATYPES = { + "double": Float, + "scaled": Float, # TODO: + "int": Int, + "bool": Bool, + "enum": Enum, + "string": String, + "blob": Waveform, # TODO: waveform[u8] really + "array": Waveform, # TODO: specify generic type + "tuple": Table, # Table of anonymous arrays (all of which happen to have length 1) + "struct": Table, # Table of named arrays (all of which happen to have length 1) + "matrix": Waveform, # Maybe? + "command": ..., # Special treatment - it's an AttrX +} + + class SecopError(Exception): pass @@ -31,21 +54,45 @@ def __init__(self, *, connection: IPConnection) -> None: async def update(self, attr: AttrR[NumberT, SecopAttributeIORef]) -> None: """Read value from device and update the value in FastCS.""" - query = f"read {attr.io_ref.module_name}:{attr.io_ref.accessible_name}\n" - response = await self._connection.send_query(query) - response = response.strip() + try: + query = f"read {attr.io_ref.module_name}:{attr.io_ref.accessible_name}\n" + response = await self._connection.send_query(query) + response = response.strip() + + prefix = f"reply {attr.io_ref.module_name}:{attr.io_ref.accessible_name} " + if not response.startswith(prefix): + raise SecopError( + f"Invalid response to 'read' command by SECoP device: '{response}'" + ) + + value = json.loads(response[len(prefix) :]) + + await attr.update(attr.dtype(value[0])) + except ConnectionError: + # Reconnect will be attempted in a periodic scan task + pass + except Exception: + logger.exception("Exception during update()") - prefix = f"reply {attr.io_ref.module_name}:{attr.io_ref.accessible_name} " - if not response.startswith(prefix): - raise SecopError(f"Invalid response to 'read' command by SECoP device: '{response}") + async def send(self, attr: AttrW[NumberT, SecopAttributeIORef], value: NumberT) -> None: + """Send a value from FastCS to the device.""" + try: + query = f"change {attr.io_ref.module_name}:{attr.io_ref.accessible_name} {value}\n" - value = json.loads(response[len(prefix) :]) + response = await self._connection.send_query(query) + response = response.strip() - await attr.update(attr.dtype(value[0])) + prefix = f"changed {attr.io_ref.module_name}:{attr.io_ref.accessible_name} " - async def send(self, attr: AttrW[NumberT, SecopAttributeIORef], value: NumberT) -> None: - """Send a value from FastCS to the device.""" - raise NotImplementedError() + if not response.startswith(prefix): + raise SecopError( + f"Invalid response to 'change' command by SECoP device: '{response}'" + ) + except ConnectionError: + # Reconnect will be attempted in a periodic scan task + pass + except Exception: + logger.exception("Exception during update()") class SecopModuleController(Controller): @@ -61,29 +108,21 @@ def __init__( self._module = module super().__init__(ios=[self._io]) - def parameter_descriptor_to_attribute( - self, - module_name: str, - accessible_name: str, - parameter_descriptor: dict[str, typing.Any], - ) -> Attribute: - return AttrR( - Float(), - io_ref=SecopAttributeIORef( - module_name=module_name, - accessible_name=accessible_name, - update_period=0.1, - ), - ) - async def initialise(self) -> None: for parameter_name, parameter in self._module["accessibles"].items(): + # secop_dtype = parameter["type"] + # if secop_dtype == "command": + # self.add_attribute() + self.add_attribute( parameter_name, - self.parameter_descriptor_to_attribute( - module_name=self._module_name, - accessible_name=parameter_name, - parameter_descriptor=parameter, + AttrRW( + Float(), + io_ref=SecopAttributeIORef( + module_name=self._module_name, + accessible_name=parameter_name, + update_period=1, + ), ), ) @@ -98,6 +137,18 @@ def __init__(self, settings: IPConnectionSettings) -> None: async def connect(self) -> None: await self._connection.connect(self._ip_settings) + @scan(15.0) + async def attempt_reconnect_if_sending_idn_fails(self) -> None: + try: + await self._connection.send_query("*IDN?\n") + except ConnectionError: + logger.info("Detected connection loss, attempting reconnect.") + try: + await self.connect() + logger.info("Reconnect successful.") + except Exception: + logger.info("Reconnect failed.") + async def check_idn(self) -> None: """ Checks that the response to *IDN? indicates this is a SECoP device. @@ -124,12 +175,15 @@ async def check_idn(self) -> None: f"Not a SECoP device?" ) - print(f"Connected to SECoP device with IDN='{identification}'") + print(f"Connected to SECoP device with IDN='{identification}'.") async def initialise(self) -> None: await self.connect() await self.check_idn() + # Turn off asynchronous replies. + await self._connection.send_query("deactivate\n") + descriptor = await self._connection.send_query("describe\n") if not descriptor.startswith("describing . "): raise SecopError(f"Invalid response to 'describe': '{descriptor}'.") @@ -154,6 +208,10 @@ async def initialise(self) -> None: if __name__ == "__main__": + configure_logging(level=LogLevel.DEBUG) + + logging.basicConfig(level=LogLevel.DEBUG) + epics_options = EpicsIOCOptions(pv_prefix="TE:NDW2922:SECOP") epics_ca = EpicsCATransport(epicsca=epics_options) epics_pva = EpicsPVATransport(epicspva=epics_options) @@ -162,4 +220,4 @@ async def initialise(self) -> None: SecopController(settings=IPConnectionSettings(ip="127.0.0.1", port=57677)), [epics_ca], ) - fastcs.run() + fastcs.run(interactive=True) diff --git a/tests/emulators/simple_secop/interfaces/stream_interface.py b/tests/emulators/simple_secop/interfaces/stream_interface.py index 9a5c1df..7394b50 100644 --- a/tests/emulators/simple_secop/interfaces/stream_interface.py +++ b/tests/emulators/simple_secop/interfaces/stream_interface.py @@ -9,8 +9,9 @@ @has_log class SimpleSecopStreamInterface(StreamInterface): commands = { - # ID CmdBuilder("idn").escape("*IDN?").optional("\r").eos().build(), + CmdBuilder("deactivate").escape("deactivate").optional(" .").optional("\r").eos().build(), + CmdBuilder("activate").escape("activate").optional(" .").optional("\r").eos().build(), CmdBuilder("describe").escape("describe").optional("\r").eos().build(), CmdBuilder("change") .escape("change ") @@ -57,3 +58,9 @@ def change(self, module: str, accessible: str, value: typing.Any): def read(self, module: str, accessible: str): data_report = self._device.modules[module].accessibles[accessible].data_report() return f"reply {module}:{accessible} {json.dumps(data_report)}" + + def deactivate(self): + return "inactive" + + def activate(self): + raise ValueError("emulator does not (yet) support sending asynchronous updates.") diff --git a/tests/test_against_emulator.py b/tests/test_against_emulator.py index ff57a6b..00c8d9d 100644 --- a/tests/test_against_emulator.py +++ b/tests/test_against_emulator.py @@ -29,7 +29,8 @@ def emulator(): proc.kill() -async def test_sub_controllers_created(): +@pytest.fixture +async def controller(): controller = SecopController( settings=IPConnectionSettings( ip="127.0.0.1", @@ -39,7 +40,17 @@ async def test_sub_controllers_created(): await controller.connect() await controller.initialise() + return controller + +async def test_sub_controllers_created(controller): assert "mod1" in controller.sub_controllers assert "mod2" in controller.sub_controllers assert "mod3" in controller.sub_controllers + + +async def test_attributes_created(controller): + for mod in ["mod1", "mod2", "mod3"]: + assert "p1" in controller.sub_controllers[mod].attributes + assert "p2" in controller.sub_controllers[mod].attributes + assert "p3" in controller.sub_controllers[mod].attributes From afc857817ed18a241d5ed2d19e445c70773659ba Mon Sep 17 00:00:00 2001 From: Tom Willemsen Date: Fri, 19 Dec 2025 10:48:46 +0000 Subject: [PATCH 03/19] Proof-of-concept for EGU/PREC/DESC --- src/fastcs_secop/__init__.py | 49 ++++++++++++++++++-------- tests/emulators/simple_secop/device.py | 16 ++++++--- tests/test_utils.py | 9 +++++ 3 files changed, 55 insertions(+), 19 deletions(-) create mode 100644 tests/test_utils.py diff --git a/src/fastcs_secop/__init__.py b/src/fastcs_secop/__init__.py index 8c402e3..ba08b53 100644 --- a/src/fastcs_secop/__init__.py +++ b/src/fastcs_secop/__init__.py @@ -46,6 +46,19 @@ class SecopAttributeIORef(AttributeIORef): accessible_name: str = "" +def format_string_to_prec(fmt_str: str | None) -> int | None: + """ + Convert a SECoP format-string specifier to a precision. + """ + if fmt_str is None: + return None + + if fmt_str.startswith("%.") and fmt_str.endswith("f"): + return int(fmt_str[2:-1]) + + return None + + class SecopAttributeIO(AttributeIO[NumberT, SecopAttributeIORef]): def __init__(self, *, connection: IPConnection) -> None: super().__init__() @@ -110,21 +123,29 @@ def __init__( async def initialise(self) -> None: for parameter_name, parameter in self._module["accessibles"].items(): - # secop_dtype = parameter["type"] - # if secop_dtype == "command": - # self.add_attribute() - - self.add_attribute( - parameter_name, - AttrRW( - Float(), - io_ref=SecopAttributeIORef( - module_name=self._module_name, - accessible_name=parameter_name, - update_period=1, + datainfo = parameter["datainfo"] + secop_dtype = datainfo["type"] + if secop_dtype == "command": + # TODO: handle commands + pass + elif secop_dtype == "double": + self.add_attribute( + parameter_name, + AttrRW( + Float( + units=datainfo.get("unit", None), + min_alarm=datainfo.get("min", None), + max_alarm=datainfo.get("max", None), + prec=format_string_to_prec(datainfo.get("fmtstr", None)) or 6, + ), + io_ref=SecopAttributeIORef( + module_name=self._module_name, + accessible_name=parameter_name, + update_period=1, + ), + description=parameter.get("description", ""), ), - ), - ) + ) class SecopController(Controller): diff --git a/tests/emulators/simple_secop/device.py b/tests/emulators/simple_secop/device.py index 25c9658..6ec28c9 100644 --- a/tests/emulators/simple_secop/device.py +++ b/tests/emulators/simple_secop/device.py @@ -13,9 +13,13 @@ def __init__(self): class Parameter(Accessible): - def __init__(self, value): + def __init__(self, value, dtype="double", unit="", prec=3, desc=""): super().__init__() self.value = value + self.dtype = dtype + self.unit = unit + self.prec = prec + self.desc = desc def data_report(self): return [ @@ -27,9 +31,11 @@ def data_report(self): def descriptor(self) -> dict[str, typing.Any]: return { - "description": "some_parameter_description", + "description": self.desc, "datainfo": { "type": "double", + "fmtstr": f"%.{self.prec}f", + "unit": self.unit, }, "readonly": False, } @@ -51,9 +57,9 @@ def descriptor(self) -> dict[str, typing.Any]: class Module: def __init__(self): self.accessibles = { - "p1": Parameter(1), - "p2": Parameter(2), - "p3": Parameter(3), + "p1": Parameter(1, unit="mm", prec=2, desc="parameter one"), + "p2": Parameter(2, unit="uA", prec=4, desc="parameter two"), + "p3": Parameter(3, unit="kV", prec=1, desc="parameter three"), } self.description = "foo" diff --git a/tests/test_utils.py b/tests/test_utils.py new file mode 100644 index 0000000..316202e --- /dev/null +++ b/tests/test_utils.py @@ -0,0 +1,9 @@ +from fastcs_secop import format_string_to_prec + + +def test_format_string_to_prec(): + assert format_string_to_prec("%.1f") == 1 + assert format_string_to_prec("%.99f") == 99 + assert format_string_to_prec("%.5g") is None + assert format_string_to_prec("%.5e") is None + assert format_string_to_prec(None) is None From bf8afae3d98bedde382e73be85b71901fcc9f1ec Mon Sep 17 00:00:00 2001 From: Tom Willemsen Date: Fri, 19 Dec 2025 12:25:46 +0000 Subject: [PATCH 04/19] Add support for most datatypes --- src/fastcs_secop/__init__.py | 116 ++++++++++++++++-- tests/emulators/simple_secop/device.py | 63 ++++++++-- .../interfaces/stream_interface.py | 5 + 3 files changed, 165 insertions(+), 19 deletions(-) diff --git a/src/fastcs_secop/__init__.py b/src/fastcs_secop/__init__.py index ba08b53..8d1ae59 100644 --- a/src/fastcs_secop/__init__.py +++ b/src/fastcs_secop/__init__.py @@ -1,9 +1,14 @@ +import base64 +import enum import json import logging +import math import typing +import uuid from dataclasses import dataclass from logging import getLogger +import numpy as np from fastcs.attributes import AttributeIO, AttributeIORef, AttrR, AttrRW, AttrW from fastcs.connections import IPConnection, IPConnectionSettings from fastcs.controllers import Controller @@ -44,6 +49,8 @@ class SecopError(Exception): class SecopAttributeIORef(AttributeIORef): module_name: str = "" accessible_name: str = "" + decode: callable = lambda x: x + encode: callable = lambda x: x def format_string_to_prec(fmt_str: str | None) -> int | None: @@ -78,9 +85,11 @@ async def update(self, attr: AttrR[NumberT, SecopAttributeIORef]) -> None: f"Invalid response to 'read' command by SECoP device: '{response}'" ) - value = json.loads(response[len(prefix) :]) + value = json.loads(response[len(prefix) :])[0] - await attr.update(attr.dtype(value[0])) + value = attr.io_ref.decode(value) + + await attr.update(value) except ConnectionError: # Reconnect will be attempted in a periodic scan task pass @@ -125,23 +134,108 @@ async def initialise(self) -> None: for parameter_name, parameter in self._module["accessibles"].items(): datainfo = parameter["datainfo"] secop_dtype = datainfo["type"] + + min_val = datainfo.get("min") + max_val = datainfo.get("max") + scale = datainfo.get("scale") + if secop_dtype == "command": # TODO: handle commands pass - elif secop_dtype == "double": + elif secop_dtype in ["double", "scaled"]: + if min_val is not None and scale is not None: + min_val *= scale + if max_val is not None and scale is not None: + max_val *= scale + self.add_attribute( parameter_name, AttrRW( Float( units=datainfo.get("unit", None), - min_alarm=datainfo.get("min", None), - max_alarm=datainfo.get("max", None), + min_alarm=min_val, + max_alarm=max_val, prec=format_string_to_prec(datainfo.get("fmtstr", None)) or 6, ), io_ref=SecopAttributeIORef( module_name=self._module_name, accessible_name=parameter_name, - update_period=1, + update_period=1.0, + decode=lambda x: x * scale if scale is not None else x, + encode=lambda x: int(math.round(x / scale)) if scale is not None else x, + ), + description=parameter.get("description", ""), + ), + ) + elif secop_dtype == "int": + self.add_attribute( + parameter_name, + AttrRW( + Int( + units=datainfo.get("unit", None), + min_alarm=min_val, + max_alarm=max_val, + ), + io_ref=SecopAttributeIORef( + module_name=self._module_name, + accessible_name=parameter_name, + update_period=1.0, + ), + description=parameter.get("description", ""), + ), + ) + elif secop_dtype == "bool": + self.add_attribute( + parameter_name, + AttrRW( + Bool(), + io_ref=SecopAttributeIORef( + module_name=self._module_name, + accessible_name=parameter_name, + update_period=1.0, + ), + description=parameter.get("description", ""), + ), + ) + elif secop_dtype == "enum": + enum_type = enum.Enum("enum_type", datainfo["members"]) + + self.add_attribute( + parameter_name, + AttrRW( + Enum(enum_type), + io_ref=SecopAttributeIORef( + module_name=self._module_name, + accessible_name=parameter_name, + update_period=1.0, + ), + description=parameter.get("description", ""), + ), + ) + elif secop_dtype == "string": + self.add_attribute( + parameter_name, + AttrRW( + String(), + io_ref=SecopAttributeIORef( + module_name=self._module_name, + accessible_name=parameter_name, + update_period=1.0, + ), + description=parameter.get("description", ""), + ), + ) + elif secop_dtype == "blob": + self.add_attribute( + parameter_name, + AttrRW( + Waveform(np.uint8, shape=(datainfo["maxbytes"],)), + io_ref=SecopAttributeIORef( + module_name=self._module_name, + accessible_name=parameter_name, + update_period=1.0, + decode=lambda x: np.frombuffer(base64.b64decode(x), dtype=np.uint8), + encode=base64.b64encode, ), description=parameter.get("description", ""), ), @@ -159,9 +253,15 @@ async def connect(self) -> None: await self._connection.connect(self._ip_settings) @scan(15.0) - async def attempt_reconnect_if_sending_idn_fails(self) -> None: + async def ping(self) -> None: + """Ping the SECoP device, to check connection is still open. + + Attempts to reconnect if the connection was not open (e.g. closed + by remote end or network break). + """ try: - await self._connection.send_query("*IDN?\n") + token = uuid.uuid4() + await self._connection.send_query(f"ping {token}\n") except ConnectionError: logger.info("Detected connection loss, attempting reconnect.") try: diff --git a/tests/emulators/simple_secop/device.py b/tests/emulators/simple_secop/device.py index 6ec28c9..1c4ca1b 100644 --- a/tests/emulators/simple_secop/device.py +++ b/tests/emulators/simple_secop/device.py @@ -1,3 +1,4 @@ +import base64 import time import typing from collections import OrderedDict @@ -13,17 +14,29 @@ def __init__(self): class Parameter(Accessible): - def __init__(self, value, dtype="double", unit="", prec=3, desc=""): + def __init__( + self, + value, + *, + dtype="double", + unit="", + prec=3, + desc="", + extra_datainfo: dict[str, typing.Any] | None = None, + value_encoder=lambda x: x, + ): super().__init__() self.value = value self.dtype = dtype self.unit = unit self.prec = prec self.desc = desc + self.extra_datainfo = extra_datainfo or {} + self.value_encoder = value_encoder def data_report(self): return [ - self.value, + self.value_encoder(self.value), { "t": time.time(), }, @@ -33,9 +46,10 @@ def descriptor(self) -> dict[str, typing.Any]: return { "description": self.desc, "datainfo": { - "type": "double", + "type": self.dtype, "fmtstr": f"%.{self.prec}f", "unit": self.unit, + **self.extra_datainfo, }, "readonly": False, } @@ -54,14 +68,41 @@ def descriptor(self) -> dict[str, typing.Any]: } -class Module: +class OneOfEachDtypeModule: def __init__(self): self.accessibles = { - "p1": Parameter(1, unit="mm", prec=2, desc="parameter one"), - "p2": Parameter(2, unit="uA", prec=4, desc="parameter two"), - "p3": Parameter(3, unit="kV", prec=1, desc="parameter three"), + "double": Parameter( + 1.2345, unit="mm", prec=2, desc="a double parameter", dtype="double" + ), + "scaled": Parameter( + 42, + unit="uA", + prec=4, + desc="a scaled parameter", + dtype="scaled", + extra_datainfo={"scale": 47}, + ), + "int": Parameter(73, desc="an integer parameter", dtype="int"), + "bool": Parameter(True, desc="a boolean parameter", dtype="bool"), + "enum": Parameter( + 1, + desc="an enum parameter", + dtype="enum", + extra_datainfo={ + "members": {"zero": 0, "one": 1, "two": 2, "three": 3, "four": 4, "five": 5} + }, + ), + "string": Parameter("hello", desc="a string parameter", dtype="string"), + "blob": Parameter( + b"a blob of binary data", + desc="a blob parameter", + dtype="blob", + value_encoder=lambda x: base64.b64encode(x).decode("ascii"), + extra_datainfo={"maxbytes": 512}, + ), } - self.description = "foo" + + self.description = "a module with one accessible of each possible dtype" def descriptor(self) -> dict[str, typing.Any]: return { @@ -78,9 +119,9 @@ class SimulatedSecopNode(StateMachineDevice): def _initialize_data(self): """Initialize the device's attributes.""" self.modules = { - "mod1": Module(), - "mod2": Module(), - "mod3": Module(), + "mod1": OneOfEachDtypeModule(), + "mod2": OneOfEachDtypeModule(), + "mod3": OneOfEachDtypeModule(), } def _get_state_handlers(self): diff --git a/tests/emulators/simple_secop/interfaces/stream_interface.py b/tests/emulators/simple_secop/interfaces/stream_interface.py index 7394b50..83d4080 100644 --- a/tests/emulators/simple_secop/interfaces/stream_interface.py +++ b/tests/emulators/simple_secop/interfaces/stream_interface.py @@ -1,4 +1,5 @@ import json +import time import typing from lewis.adapters.stream import StreamInterface @@ -10,6 +11,7 @@ class SimpleSecopStreamInterface(StreamInterface): commands = { CmdBuilder("idn").escape("*IDN?").optional("\r").eos().build(), + CmdBuilder("ping").escape("ping ").any().optional("\r").eos().build(), CmdBuilder("deactivate").escape("deactivate").optional(" .").optional("\r").eos().build(), CmdBuilder("activate").escape("activate").optional(" .").optional("\r").eos().build(), CmdBuilder("describe").escape("describe").optional("\r").eos().build(), @@ -47,6 +49,9 @@ def handle_error(self, request, error): def idn(self): return "ISSE&SINE2020,SECoP,V0000.00.00,lewis_emulator" + def ping(self, token): + return f"pong {token} {json.dumps([None, {'t': time.time()}])}" + def describe(self): return f"describing . {json.dumps(self._device.descriptor())}" From f8aed03e3d49c36c245f32f3d8eddcddae45e8fe Mon Sep 17 00:00:00 2001 From: Tom Willemsen Date: Mon, 22 Dec 2025 13:23:08 +0000 Subject: [PATCH 05/19] array support --- src/fastcs_secop/__init__.py | 59 +++++++++++++++++--------- tests/emulators/simple_secop/device.py | 34 ++++++++++----- tests/test_against_emulator.py | 26 ++++++++---- 3 files changed, 80 insertions(+), 39 deletions(-) diff --git a/src/fastcs_secop/__init__.py b/src/fastcs_secop/__init__.py index 8d1ae59..406f4d8 100644 --- a/src/fastcs_secop/__init__.py +++ b/src/fastcs_secop/__init__.py @@ -2,17 +2,17 @@ import enum import json import logging -import math import typing import uuid from dataclasses import dataclass from logging import getLogger import numpy as np +import numpy.typing as npt from fastcs.attributes import AttributeIO, AttributeIORef, AttrR, AttrRW, AttrW from fastcs.connections import IPConnection, IPConnectionSettings from fastcs.controllers import Controller -from fastcs.datatypes import Bool, Enum, Float, Int, String, Table, Waveform +from fastcs.datatypes import Bool, Enum, Float, Int, String, Waveform from fastcs.launch import FastCS from fastcs.logging import LogLevel, configure_logging from fastcs.methods import scan @@ -25,22 +25,6 @@ logger = getLogger(__name__) -SECOP_DATATYPES = { - "double": Float, - "scaled": Float, # TODO: - "int": Int, - "bool": Bool, - "enum": Enum, - "string": String, - "blob": Waveform, # TODO: waveform[u8] really - "array": Waveform, # TODO: specify generic type - "tuple": Table, # Table of anonymous arrays (all of which happen to have length 1) - "struct": Table, # Table of named arrays (all of which happen to have length 1) - "matrix": Waveform, # Maybe? - "command": ..., # Special treatment - it's an AttrX -} - - class SecopError(Exception): pass @@ -66,6 +50,17 @@ def format_string_to_prec(fmt_str: str | None) -> int | None: return None +def secop_dtype_to_numpy_dtype(secop_dtype: str) -> str: + if secop_dtype == "double": + return "float64" + elif secop_dtype == "int": + return "int32" + elif secop_dtype == "bool": + return "int32" + else: + raise SecopError(f"Cannot handle SECoP dtype '{secop_dtype}' within array/struct/tuple") + + class SecopAttributeIO(AttributeIO[NumberT, SecopAttributeIORef]): def __init__(self, *, connection: IPConnection) -> None: super().__init__() @@ -86,9 +81,7 @@ async def update(self, attr: AttrR[NumberT, SecopAttributeIORef]) -> None: ) value = json.loads(response[len(prefix) :])[0] - value = attr.io_ref.decode(value) - await attr.update(value) except ConnectionError: # Reconnect will be attempted in a periodic scan task @@ -162,7 +155,7 @@ async def initialise(self) -> None: accessible_name=parameter_name, update_period=1.0, decode=lambda x: x * scale if scale is not None else x, - encode=lambda x: int(math.round(x / scale)) if scale is not None else x, + encode=lambda x: int(round(x / scale)) if scale is not None else x, ), description=parameter.get("description", ""), ), @@ -198,6 +191,7 @@ async def initialise(self) -> None: ), ) elif secop_dtype == "enum": + # TODO: Bug - this doesn't work properly with PVA? enum_type = enum.Enum("enum_type", datainfo["members"]) self.add_attribute( @@ -240,6 +234,29 @@ async def initialise(self) -> None: description=parameter.get("description", ""), ), ) + elif secop_dtype == "array": + inner_dtype = datainfo["members"]["type"] + if inner_dtype not in ["double", "int", "bool"]: + raise SecopError(f"Cannot handle inner dtype {inner_dtype} in array.") + + np_inner_dtype = secop_dtype_to_numpy_dtype(inner_dtype) + + def decode(x: list[int | float], t: npt.DTypeLike = np_inner_dtype) -> npt.NDArray: + return np.array(x, dtype=t) + + self.add_attribute( + parameter_name, + AttrRW( + Waveform(np_inner_dtype, shape=(datainfo["maxlen"],)), + io_ref=SecopAttributeIORef( + module_name=self._module_name, + accessible_name=parameter_name, + update_period=1.0, + decode=decode, + ), + description=parameter.get("description", ""), + ), + ) class SecopController(Controller): diff --git a/tests/emulators/simple_secop/device.py b/tests/emulators/simple_secop/device.py index 1c4ca1b..1c1f00e 100644 --- a/tests/emulators/simple_secop/device.py +++ b/tests/emulators/simple_secop/device.py @@ -85,12 +85,10 @@ def __init__(self): "int": Parameter(73, desc="an integer parameter", dtype="int"), "bool": Parameter(True, desc="a boolean parameter", dtype="bool"), "enum": Parameter( - 1, + 3, desc="an enum parameter", dtype="enum", - extra_datainfo={ - "members": {"zero": 0, "one": 1, "two": 2, "three": 3, "four": 4, "five": 5} - }, + extra_datainfo={"members": {"one": 1, "two": 2, "three": 3, "four": 4, "five": 5}}, ), "string": Parameter("hello", desc="a string parameter", dtype="string"), "blob": Parameter( @@ -100,6 +98,24 @@ def __init__(self): value_encoder=lambda x: base64.b64encode(x).decode("ascii"), extra_datainfo={"maxbytes": 512}, ), + "double_array": Parameter( + [1.414, 1.618, 2.718, 3.14159], + desc="a double array parameter", + dtype="array", + extra_datainfo={"maxlen": 512, "members": {"type": "double"}}, + ), + "int_array": Parameter( + [1, 1, 2, 3, 5, 8, 13], + desc="an integer array parameter", + dtype="array", + extra_datainfo={"maxlen": 512, "members": {"type": "int"}}, + ), + "bool_array": Parameter( + [True, True, False, True, False, False, True, True], + desc="a bool array parameter", + dtype="array", + extra_datainfo={"maxlen": 512, "members": {"type": "bool"}}, + ), } self.description = "a module with one accessible of each possible dtype" @@ -108,7 +124,7 @@ def descriptor(self) -> dict[str, typing.Any]: return { "implementation": __name__, "description": self.description, - "interface_classes": ["Readable"], + "interface_classes": [], "accessibles": { name: accessible.descriptor() for name, accessible in self.accessibles.items() }, @@ -119,9 +135,7 @@ class SimulatedSecopNode(StateMachineDevice): def _initialize_data(self): """Initialize the device's attributes.""" self.modules = { - "mod1": OneOfEachDtypeModule(), - "mod2": OneOfEachDtypeModule(), - "mod3": OneOfEachDtypeModule(), + "one_of_everything": OneOfEachDtypeModule(), } def _get_state_handlers(self): @@ -135,7 +149,7 @@ def _get_transition_handlers(self): def descriptor(self) -> dict[str, typing.Any]: return { - "equipment_id": "fastcs_secop-lewis-emulator", - "description": "Simple SECoP emulator", + "equipment_id": __name__, + "description": "SECoP lewis emulator", "modules": {name: module.descriptor() for name, module in self.modules.items()}, } diff --git a/tests/test_against_emulator.py b/tests/test_against_emulator.py index 00c8d9d..f409d1d 100644 --- a/tests/test_against_emulator.py +++ b/tests/test_against_emulator.py @@ -44,13 +44,23 @@ async def controller(): async def test_sub_controllers_created(controller): - assert "mod1" in controller.sub_controllers - assert "mod2" in controller.sub_controllers - assert "mod3" in controller.sub_controllers + assert "one_of_everything" in controller.sub_controllers -async def test_attributes_created(controller): - for mod in ["mod1", "mod2", "mod3"]: - assert "p1" in controller.sub_controllers[mod].attributes - assert "p2" in controller.sub_controllers[mod].attributes - assert "p3" in controller.sub_controllers[mod].attributes +@pytest.mark.parametrize( + "param", + [ + "double", + "scaled", + "int", + "bool", + "enum", + "string", + "blob", + "int_array", + "bool_array", + "double_array", + ], +) +async def test_attributes_created(controller, param): + assert param in controller.sub_controllers["one_of_everything"].attributes From bb7ee1c16b369d1fe786087bc84b7ed45eaa4d8b Mon Sep 17 00:00:00 2001 From: Tom Willemsen Date: Tue, 23 Dec 2025 11:13:55 +0000 Subject: [PATCH 06/19] Fix tests & split controllers & io --- src/fastcs_secop/__init__.py | 344 +------------------- src/fastcs_secop/_controllers.py | 280 ++++++++++++++++ src/fastcs_secop/_io.py | 70 ++++ src/fastcs_secop/_util.py | 2 + tests/emulators/simple_secop/device.py | 2 +- tests/test_against_emulator.py | 81 ++++- tests/{test_utils.py => test_controller.py} | 2 +- 7 files changed, 427 insertions(+), 354 deletions(-) create mode 100644 src/fastcs_secop/_controllers.py create mode 100644 src/fastcs_secop/_io.py create mode 100644 src/fastcs_secop/_util.py rename tests/{test_utils.py => test_controller.py} (82%) diff --git a/src/fastcs_secop/__init__.py b/src/fastcs_secop/__init__.py index 406f4d8..361c1e5 100644 --- a/src/fastcs_secop/__init__.py +++ b/src/fastcs_secop/__init__.py @@ -1,348 +1,22 @@ -import base64 -import enum -import json import logging -import typing -import uuid -from dataclasses import dataclass from logging import getLogger -import numpy as np -import numpy.typing as npt -from fastcs.attributes import AttributeIO, AttributeIORef, AttrR, AttrRW, AttrW -from fastcs.connections import IPConnection, IPConnectionSettings -from fastcs.controllers import Controller -from fastcs.datatypes import Bool, Enum, Float, Int, String, Waveform +from fastcs.connections import IPConnectionSettings from fastcs.launch import FastCS from fastcs.logging import LogLevel, configure_logging -from fastcs.methods import scan from fastcs.transports import EpicsIOCOptions, EpicsPVATransport from fastcs.transports.epics.ca import EpicsCATransport -NumberT = typing.TypeVar("NumberT", int, float) - +from fastcs_secop._controllers import SecopController, SecopModuleController +from fastcs_secop._util import SecopError logger = getLogger(__name__) - -class SecopError(Exception): - pass - - -@dataclass -class SecopAttributeIORef(AttributeIORef): - module_name: str = "" - accessible_name: str = "" - decode: callable = lambda x: x - encode: callable = lambda x: x - - -def format_string_to_prec(fmt_str: str | None) -> int | None: - """ - Convert a SECoP format-string specifier to a precision. - """ - if fmt_str is None: - return None - - if fmt_str.startswith("%.") and fmt_str.endswith("f"): - return int(fmt_str[2:-1]) - - return None - - -def secop_dtype_to_numpy_dtype(secop_dtype: str) -> str: - if secop_dtype == "double": - return "float64" - elif secop_dtype == "int": - return "int32" - elif secop_dtype == "bool": - return "int32" - else: - raise SecopError(f"Cannot handle SECoP dtype '{secop_dtype}' within array/struct/tuple") - - -class SecopAttributeIO(AttributeIO[NumberT, SecopAttributeIORef]): - def __init__(self, *, connection: IPConnection) -> None: - super().__init__() - - self._connection = connection - - async def update(self, attr: AttrR[NumberT, SecopAttributeIORef]) -> None: - """Read value from device and update the value in FastCS.""" - try: - query = f"read {attr.io_ref.module_name}:{attr.io_ref.accessible_name}\n" - response = await self._connection.send_query(query) - response = response.strip() - - prefix = f"reply {attr.io_ref.module_name}:{attr.io_ref.accessible_name} " - if not response.startswith(prefix): - raise SecopError( - f"Invalid response to 'read' command by SECoP device: '{response}'" - ) - - value = json.loads(response[len(prefix) :])[0] - value = attr.io_ref.decode(value) - await attr.update(value) - except ConnectionError: - # Reconnect will be attempted in a periodic scan task - pass - except Exception: - logger.exception("Exception during update()") - - async def send(self, attr: AttrW[NumberT, SecopAttributeIORef], value: NumberT) -> None: - """Send a value from FastCS to the device.""" - try: - query = f"change {attr.io_ref.module_name}:{attr.io_ref.accessible_name} {value}\n" - - response = await self._connection.send_query(query) - response = response.strip() - - prefix = f"changed {attr.io_ref.module_name}:{attr.io_ref.accessible_name} " - - if not response.startswith(prefix): - raise SecopError( - f"Invalid response to 'change' command by SECoP device: '{response}'" - ) - except ConnectionError: - # Reconnect will be attempted in a periodic scan task - pass - except Exception: - logger.exception("Exception during update()") - - -class SecopModuleController(Controller): - def __init__( - self, - *, - connection: IPConnection, - module_name: str, - module: dict[str, typing.Any], - ) -> None: - self._io = SecopAttributeIO(connection=connection) - self._module_name = module_name - self._module = module - super().__init__(ios=[self._io]) - - async def initialise(self) -> None: - for parameter_name, parameter in self._module["accessibles"].items(): - datainfo = parameter["datainfo"] - secop_dtype = datainfo["type"] - - min_val = datainfo.get("min") - max_val = datainfo.get("max") - scale = datainfo.get("scale") - - if secop_dtype == "command": - # TODO: handle commands - pass - elif secop_dtype in ["double", "scaled"]: - if min_val is not None and scale is not None: - min_val *= scale - if max_val is not None and scale is not None: - max_val *= scale - - self.add_attribute( - parameter_name, - AttrRW( - Float( - units=datainfo.get("unit", None), - min_alarm=min_val, - max_alarm=max_val, - prec=format_string_to_prec(datainfo.get("fmtstr", None)) or 6, - ), - io_ref=SecopAttributeIORef( - module_name=self._module_name, - accessible_name=parameter_name, - update_period=1.0, - decode=lambda x: x * scale if scale is not None else x, - encode=lambda x: int(round(x / scale)) if scale is not None else x, - ), - description=parameter.get("description", ""), - ), - ) - elif secop_dtype == "int": - self.add_attribute( - parameter_name, - AttrRW( - Int( - units=datainfo.get("unit", None), - min_alarm=min_val, - max_alarm=max_val, - ), - io_ref=SecopAttributeIORef( - module_name=self._module_name, - accessible_name=parameter_name, - update_period=1.0, - ), - description=parameter.get("description", ""), - ), - ) - elif secop_dtype == "bool": - self.add_attribute( - parameter_name, - AttrRW( - Bool(), - io_ref=SecopAttributeIORef( - module_name=self._module_name, - accessible_name=parameter_name, - update_period=1.0, - ), - description=parameter.get("description", ""), - ), - ) - elif secop_dtype == "enum": - # TODO: Bug - this doesn't work properly with PVA? - enum_type = enum.Enum("enum_type", datainfo["members"]) - - self.add_attribute( - parameter_name, - AttrRW( - Enum(enum_type), - io_ref=SecopAttributeIORef( - module_name=self._module_name, - accessible_name=parameter_name, - update_period=1.0, - ), - description=parameter.get("description", ""), - ), - ) - elif secop_dtype == "string": - self.add_attribute( - parameter_name, - AttrRW( - String(), - io_ref=SecopAttributeIORef( - module_name=self._module_name, - accessible_name=parameter_name, - update_period=1.0, - ), - description=parameter.get("description", ""), - ), - ) - elif secop_dtype == "blob": - self.add_attribute( - parameter_name, - AttrRW( - Waveform(np.uint8, shape=(datainfo["maxbytes"],)), - io_ref=SecopAttributeIORef( - module_name=self._module_name, - accessible_name=parameter_name, - update_period=1.0, - decode=lambda x: np.frombuffer(base64.b64decode(x), dtype=np.uint8), - encode=base64.b64encode, - ), - description=parameter.get("description", ""), - ), - ) - elif secop_dtype == "array": - inner_dtype = datainfo["members"]["type"] - if inner_dtype not in ["double", "int", "bool"]: - raise SecopError(f"Cannot handle inner dtype {inner_dtype} in array.") - - np_inner_dtype = secop_dtype_to_numpy_dtype(inner_dtype) - - def decode(x: list[int | float], t: npt.DTypeLike = np_inner_dtype) -> npt.NDArray: - return np.array(x, dtype=t) - - self.add_attribute( - parameter_name, - AttrRW( - Waveform(np_inner_dtype, shape=(datainfo["maxlen"],)), - io_ref=SecopAttributeIORef( - module_name=self._module_name, - accessible_name=parameter_name, - update_period=1.0, - decode=decode, - ), - description=parameter.get("description", ""), - ), - ) - - -class SecopController(Controller): - def __init__(self, settings: IPConnectionSettings) -> None: - self._ip_settings = settings - self._connection = IPConnection() - - super().__init__() - - async def connect(self) -> None: - await self._connection.connect(self._ip_settings) - - @scan(15.0) - async def ping(self) -> None: - """Ping the SECoP device, to check connection is still open. - - Attempts to reconnect if the connection was not open (e.g. closed - by remote end or network break). - """ - try: - token = uuid.uuid4() - await self._connection.send_query(f"ping {token}\n") - except ConnectionError: - logger.info("Detected connection loss, attempting reconnect.") - try: - await self.connect() - logger.info("Reconnect successful.") - except Exception: - logger.info("Reconnect failed.") - - async def check_idn(self) -> None: - """ - Checks that the response to *IDN? indicates this is a SECoP device. - - Raises: - ValueError: if device is not a SECoP device. - """ - identification = await self._connection.send_query("*IDN?\n") - identification = identification.strip() - - manufacturer, product, draft_date, version = identification.split(",") - if manufacturer not in [ - "ISSE&SINE2020", # SECOP 1.x - "ISSE", # SECOP 2.x - ]: - raise SecopError( - f"Device responded to '*IDN?' with bad manufacturer string '{manufacturer}'. " - f"Not a SECoP device?" - ) - - if product != "SECoP": - raise SecopError( - f"Device responded to '*IDN?' with bad product string '{product}'. " - f"Not a SECoP device?" - ) - - print(f"Connected to SECoP device with IDN='{identification}'.") - - async def initialise(self) -> None: - await self.connect() - await self.check_idn() - - # Turn off asynchronous replies. - await self._connection.send_query("deactivate\n") - - descriptor = await self._connection.send_query("describe\n") - if not descriptor.startswith("describing . "): - raise SecopError(f"Invalid response to 'describe': '{descriptor}'.") - - descriptor = json.loads(descriptor[len("describing . ") :]) - - description = descriptor["description"] - equipment_id = descriptor["equipment_id"] - - print(f"SECoP equipment_id = '{equipment_id}', description = '{description}'") - - modules = descriptor["modules"] - - for module_name, module in modules.items(): - module_controller = SecopModuleController( - connection=self._connection, - module_name=module_name, - module=module, - ) - await module_controller.initialise() - self.add_sub_controller(name=module_name, sub_controller=module_controller) +__all__ = [ + "SecopController", + "SecopModuleController", + "SecopError", +] if __name__ == "__main__": @@ -358,4 +32,4 @@ async def initialise(self) -> None: SecopController(settings=IPConnectionSettings(ip="127.0.0.1", port=57677)), [epics_ca], ) - fastcs.run(interactive=True) + fastcs.run(interactive=False) diff --git a/src/fastcs_secop/_controllers.py b/src/fastcs_secop/_controllers.py new file mode 100644 index 0000000..93a7953 --- /dev/null +++ b/src/fastcs_secop/_controllers.py @@ -0,0 +1,280 @@ +import base64 +import enum +import json +import typing +import uuid +from logging import getLogger + +import numpy as np +import numpy.typing as npt +from fastcs.attributes import AttrRW +from fastcs.connections import IPConnection, IPConnectionSettings +from fastcs.controllers import Controller +from fastcs.datatypes import Bool, Enum, Float, Int, String, Waveform +from fastcs.methods import scan + +from fastcs_secop._io import SecopAttributeIO, SecopAttributeIORef +from fastcs_secop._util import SecopError + +logger = getLogger(__name__) + + +def format_string_to_prec(fmt_str: str | None) -> int | None: + """ + Convert a SECoP format-string specifier to a precision. + """ + if fmt_str is None: + return None + + if fmt_str.startswith("%.") and fmt_str.endswith("f"): + return int(fmt_str[2:-1]) + + return None + + +def secop_dtype_to_numpy_dtype(secop_dtype: str) -> str: + if secop_dtype == "double": + return "float64" + elif secop_dtype == "int": + return "int32" + elif secop_dtype == "bool": + return "int32" + else: + raise SecopError(f"Cannot handle SECoP dtype '{secop_dtype}' within array/struct/tuple") + + +class SecopModuleController(Controller): + def __init__( + self, + *, + connection: IPConnection, + module_name: str, + module: dict[str, typing.Any], + ) -> None: + self._io = SecopAttributeIO(connection=connection) + self._module_name = module_name + self._module = module + super().__init__(ios=[self._io]) + + async def initialise(self) -> None: + for parameter_name, parameter in self._module["accessibles"].items(): + datainfo = parameter["datainfo"] + secop_dtype = datainfo["type"] + + min_val = datainfo.get("min") + max_val = datainfo.get("max") + scale = datainfo.get("scale") + + if secop_dtype == "command": + # TODO: handle commands + pass + elif secop_dtype in ["double", "scaled"]: + if min_val is not None and scale is not None: + min_val *= scale + if max_val is not None and scale is not None: + max_val *= scale + + self.add_attribute( + parameter_name, + AttrRW( + Float( + units=datainfo.get("unit", None), + min_alarm=min_val, + max_alarm=max_val, + prec=format_string_to_prec(datainfo.get("fmtstr", None)) or 6, + ), + io_ref=SecopAttributeIORef( + module_name=self._module_name, + accessible_name=parameter_name, + update_period=1.0, + decode=lambda x, s=scale: x * s if s is not None else x, + encode=lambda x, s=scale: int(round(x / s)) if s is not None else x, + ), + description=parameter.get("description", ""), + ), + ) + elif secop_dtype == "int": + self.add_attribute( + parameter_name, + AttrRW( + Int( + units=datainfo.get("unit", None), + min_alarm=min_val, + max_alarm=max_val, + ), + io_ref=SecopAttributeIORef( + module_name=self._module_name, + accessible_name=parameter_name, + update_period=1.0, + ), + description=parameter.get("description", ""), + ), + ) + elif secop_dtype == "bool": + self.add_attribute( + parameter_name, + AttrRW( + Bool(), + io_ref=SecopAttributeIORef( + module_name=self._module_name, + accessible_name=parameter_name, + update_period=1.0, + ), + description=parameter.get("description", ""), + ), + ) + elif secop_dtype == "enum": + # TODO: Bug - this doesn't work properly with PVA? + enum_type = enum.Enum("enum_type", datainfo["members"]) + + self.add_attribute( + parameter_name, + AttrRW( + Enum(enum_type), + io_ref=SecopAttributeIORef( + module_name=self._module_name, + accessible_name=parameter_name, + update_period=1.0, + ), + description=parameter.get("description", ""), + ), + ) + elif secop_dtype == "string": + self.add_attribute( + parameter_name, + AttrRW( + String(), + io_ref=SecopAttributeIORef( + module_name=self._module_name, + accessible_name=parameter_name, + update_period=1.0, + ), + description=parameter.get("description", ""), + ), + ) + elif secop_dtype == "blob": + self.add_attribute( + parameter_name, + AttrRW( + Waveform(np.uint8, shape=(datainfo["maxbytes"],)), + io_ref=SecopAttributeIORef( + module_name=self._module_name, + accessible_name=parameter_name, + update_period=1.0, + decode=lambda x: np.frombuffer(base64.b64decode(x), dtype=np.uint8), + encode=base64.b64encode, + ), + description=parameter.get("description", ""), + ), + ) + elif secop_dtype == "array": + inner_dtype = datainfo["members"]["type"] + if inner_dtype not in ["double", "int", "bool"]: + raise SecopError(f"Cannot handle inner dtype {inner_dtype} in array.") + + np_inner_dtype = secop_dtype_to_numpy_dtype(inner_dtype) + + def decode( + x: list[int | float | bool], t: npt.DTypeLike = np_inner_dtype + ) -> npt.NDArray: + return np.array(x, dtype=t) + + self.add_attribute( + parameter_name, + AttrRW( + Waveform(np_inner_dtype, shape=(datainfo["maxlen"],)), + io_ref=SecopAttributeIORef( + module_name=self._module_name, + accessible_name=parameter_name, + update_period=1.0, + decode=decode, + ), + description=parameter.get("description", ""), + ), + ) + + +class SecopController(Controller): + def __init__(self, settings: IPConnectionSettings) -> None: + self._ip_settings = settings + self._connection = IPConnection() + + super().__init__() + + async def connect(self) -> None: + await self._connection.connect(self._ip_settings) + + @scan(15.0) + async def ping(self) -> None: + """Ping the SECoP device, to check connection is still open. + + Attempts to reconnect if the connection was not open (e.g. closed + by remote end or network break). + """ + try: + token = uuid.uuid4() + await self._connection.send_query(f"ping {token}\n") + except ConnectionError: + logger.info("Detected connection loss, attempting reconnect.") + try: + await self.connect() + logger.info("Reconnect successful.") + except Exception: + logger.info("Reconnect failed.") + + async def check_idn(self) -> None: + """ + Checks that the response to *IDN? indicates this is a SECoP device. + + Raises: + ValueError: if device is not a SECoP device. + """ + identification = await self._connection.send_query("*IDN?\n") + identification = identification.strip() + + manufacturer, product, draft_date, version = identification.split(",") + if manufacturer not in [ + "ISSE&SINE2020", # SECOP 1.x + "ISSE", # SECOP 2.x + ]: + raise SecopError( + f"Device responded to '*IDN?' with bad manufacturer string '{manufacturer}'. " + f"Not a SECoP device?" + ) + + if product != "SECoP": + raise SecopError( + f"Device responded to '*IDN?' with bad product string '{product}'. " + f"Not a SECoP device?" + ) + + print(f"Connected to SECoP device with IDN='{identification}'.") + + async def initialise(self) -> None: + await self.connect() + await self.check_idn() + + # Turn off asynchronous replies. + await self._connection.send_query("deactivate\n") + + descriptor = await self._connection.send_query("describe\n") + if not descriptor.startswith("describing . "): + raise SecopError(f"Invalid response to 'describe': '{descriptor}'.") + + descriptor = json.loads(descriptor[len("describing . ") :]) + + description = descriptor["description"] + equipment_id = descriptor["equipment_id"] + + print(f"SECoP equipment_id = '{equipment_id}', description = '{description}'") + + modules = descriptor["modules"] + + for module_name, module in modules.items(): + module_controller = SecopModuleController( + connection=self._connection, + module_name=module_name, + module=module, + ) + await module_controller.initialise() + self.add_sub_controller(name=module_name, sub_controller=module_controller) diff --git a/src/fastcs_secop/_io.py b/src/fastcs_secop/_io.py new file mode 100644 index 0000000..5bc55f2 --- /dev/null +++ b/src/fastcs_secop/_io.py @@ -0,0 +1,70 @@ +import json +import typing +from dataclasses import dataclass +from logging import getLogger + +from fastcs.attributes import AttributeIO, AttributeIORef, AttrR, AttrW +from fastcs.connections import IPConnection + +from fastcs_secop._util import SecopError + +logger = getLogger(__name__) + +NumberT = typing.TypeVar("NumberT", int, float) + + +@dataclass +class SecopAttributeIORef(AttributeIORef): + module_name: str = "" + accessible_name: str = "" + decode: callable = lambda x: x + encode: callable = lambda x: x + + +class SecopAttributeIO(AttributeIO[NumberT, SecopAttributeIORef]): + def __init__(self, *, connection: IPConnection) -> None: + super().__init__() + + self._connection = connection + + async def update(self, attr: AttrR[NumberT, SecopAttributeIORef]) -> None: + """Read value from device and update the value in FastCS.""" + try: + query = f"read {attr.io_ref.module_name}:{attr.io_ref.accessible_name}\n" + response = await self._connection.send_query(query) + response = response.strip() + + prefix = f"reply {attr.io_ref.module_name}:{attr.io_ref.accessible_name} " + if not response.startswith(prefix): + raise SecopError( + f"Invalid response to 'read' command by SECoP device: '{response}'" + ) + + value = json.loads(response[len(prefix) :])[0] + value = attr.io_ref.decode(value) + await attr.update(value) + except ConnectionError: + # Reconnect will be attempted in a periodic scan task + pass + except Exception: + logger.exception("Exception during update()") + + async def send(self, attr: AttrW[NumberT, SecopAttributeIORef], value: NumberT) -> None: + """Send a value from FastCS to the device.""" + try: + query = f"change {attr.io_ref.module_name}:{attr.io_ref.accessible_name} {value}\n" + + response = await self._connection.send_query(query) + response = response.strip() + + prefix = f"changed {attr.io_ref.module_name}:{attr.io_ref.accessible_name} " + + if not response.startswith(prefix): + raise SecopError( + f"Invalid response to 'change' command by SECoP device: '{response}'" + ) + except ConnectionError: + # Reconnect will be attempted in a periodic scan task + pass + except Exception: + logger.exception("Exception during update()") diff --git a/src/fastcs_secop/_util.py b/src/fastcs_secop/_util.py new file mode 100644 index 0000000..73f935b --- /dev/null +++ b/src/fastcs_secop/_util.py @@ -0,0 +1,2 @@ +class SecopError(Exception): + pass diff --git a/tests/emulators/simple_secop/device.py b/tests/emulators/simple_secop/device.py index 1c1f00e..41a07ab 100644 --- a/tests/emulators/simple_secop/device.py +++ b/tests/emulators/simple_secop/device.py @@ -72,7 +72,7 @@ class OneOfEachDtypeModule: def __init__(self): self.accessibles = { "double": Parameter( - 1.2345, unit="mm", prec=2, desc="a double parameter", dtype="double" + 1.2345, unit="mm", prec=4, desc="a double parameter", dtype="double" ), "scaled": Parameter( 42, diff --git a/tests/test_against_emulator.py b/tests/test_against_emulator.py index f409d1d..4063060 100644 --- a/tests/test_against_emulator.py +++ b/tests/test_against_emulator.py @@ -1,14 +1,22 @@ +import asyncio import os.path import subprocess import sys +import typing +import numpy as np import pytest +from fastcs import FastCS +from fastcs.attributes import AttrR from fastcs.connections import IPConnectionSettings +from fastcs.logging import LogLevel, configure_logging from fastcs_secop import SecopController +configure_logging(level=LogLevel.TRACE) -@pytest.fixture(scope="session", autouse=True) + +@pytest.fixture(autouse=True) def emulator(): proc = subprocess.Popen( [ @@ -38,9 +46,27 @@ async def controller(): ) ) - await controller.connect() - await controller.initialise() - return controller + fastcs = FastCS( + controller, + [], + ) + + fastcs_task = asyncio.create_task(fastcs.serve(interactive=False)) + + # Wait for FastCS to have run initialise() & created attributes + max_iters = 100 # 10 seconds + for i in range(max_iters): + if controller.sub_controllers: + break + await asyncio.sleep(0.1) + else: + raise RuntimeError("No subcontrollers created within 10s of FastCS serve") + + try: + yield controller + finally: + fastcs_task.cancel() + await fastcs_task async def test_sub_controllers_created(controller): @@ -48,19 +74,40 @@ async def test_sub_controllers_created(controller): @pytest.mark.parametrize( - "param", + "param,expected_initial_value", + [ + ("double", 1.2345), + ("scaled", 42 * 47), + ("int", 73), + ("bool", True), + ("string", "hello"), + ], +) +async def test_attributes_created_for_simple_datatype(controller, param, expected_initial_value): + attr: AttrR = typing.cast( + AttrR, controller.sub_controllers["one_of_everything"].attributes[param] + ) + await attr.wait_for_value(expected_initial_value, timeout=2) + + +async def test_attributes_created_for_enum_datatype(controller): + attr: AttrR = typing.cast( + AttrR, controller.sub_controllers["one_of_everything"].attributes["enum"] + ) + await attr.wait_for_predicate(lambda v: v.name == "three", timeout=2) + + +@pytest.mark.parametrize( + "param,expected_initial_value", [ - "double", - "scaled", - "int", - "bool", - "enum", - "string", - "blob", - "int_array", - "bool_array", - "double_array", + ("blob", np.array([c for c in b"a blob of binary data"], dtype=np.uint8)), + ("int_array", np.array([1, 1, 2, 3, 5, 8, 13], dtype=np.int64)), + ("bool_array", np.array([1, 1, 0, 1, 0, 0, 1, 1], dtype=np.bool_)), + ("double_array", np.array([1.414, 1.618, 2.718, 3.14159], dtype=np.float64)), ], ) -async def test_attributes_created(controller, param): - assert param in controller.sub_controllers["one_of_everything"].attributes +async def test_attributes_created_for_array_datatype(controller, param, expected_initial_value): + attr: AttrR = typing.cast( + AttrR, controller.sub_controllers["one_of_everything"].attributes[param] + ) + await attr.wait_for_predicate(lambda v: np.array_equal(v, expected_initial_value), timeout=2) diff --git a/tests/test_utils.py b/tests/test_controller.py similarity index 82% rename from tests/test_utils.py rename to tests/test_controller.py index 316202e..91de28f 100644 --- a/tests/test_utils.py +++ b/tests/test_controller.py @@ -1,4 +1,4 @@ -from fastcs_secop import format_string_to_prec +from fastcs_secop._controllers import format_string_to_prec def test_format_string_to_prec(): From 1832cb9b1d57749f35b1012e9f55a32c1922d71f Mon Sep 17 00:00:00 2001 From: Tom Willemsen Date: Tue, 23 Dec 2025 15:02:28 +0000 Subject: [PATCH 07/19] Unit tests etc --- .github/dependabot.yml | 10 +++ .github/release.yml | 19 ++++ .github/workflows/Lint-and-test.yml | 38 ++++++++ .github/workflows/check_pr_has_label.yml | 24 +++++ .github/workflows/dependabot-prs.yml | 19 ++++ .github/workflows/documentation.yml | 21 +++++ .github/workflows/lint-and-test-nightly.yml | 9 ++ .github/workflows/release.yml | 88 +++++++++++++++++++ pyproject.toml | 4 +- ruff.toml | 53 +++++++++++ src/fastcs_secop/__init__.py | 6 +- src/fastcs_secop/_controllers.py | 31 +++---- src/fastcs_secop/_io.py | 4 +- tests/emulators/__init__.py | 1 - tests/emulators/simple_secop/device.py | 3 +- .../interfaces/stream_interface.py | 6 +- tests/test_against_emulator.py | 22 +++-- tests/test_controller.py | 43 +++++++-- 18 files changed, 358 insertions(+), 43 deletions(-) create mode 100644 .github/dependabot.yml create mode 100644 .github/release.yml create mode 100644 .github/workflows/Lint-and-test.yml create mode 100644 .github/workflows/check_pr_has_label.yml create mode 100644 .github/workflows/dependabot-prs.yml create mode 100644 .github/workflows/documentation.yml create mode 100644 .github/workflows/lint-and-test-nightly.yml create mode 100644 .github/workflows/release.yml create mode 100644 ruff.toml diff --git a/.github/dependabot.yml b/.github/dependabot.yml new file mode 100644 index 0000000..b728efb --- /dev/null +++ b/.github/dependabot.yml @@ -0,0 +1,10 @@ +version: 2 +updates: + - package-ecosystem: "pip" + directory: "/" + schedule: + interval: "daily" + - package-ecosystem: "github-actions" + directory: "/" + schedule: + interval: "daily" diff --git a/.github/release.yml b/.github/release.yml new file mode 100644 index 0000000..17b05ac --- /dev/null +++ b/.github/release.yml @@ -0,0 +1,19 @@ +changelog: + exclude: + labels: + - Semver-Ignore + categories: + - title: Breaking Changes + labels: + - Semver-Major + - breaking-change + - title: New Features + labels: + - Semver-Minor + - title: Bug Fixes + labels: + - Semver-Patch + - title: Other Changes + labels: + - Semver-Docs + - "*" diff --git a/.github/workflows/Lint-and-test.yml b/.github/workflows/Lint-and-test.yml new file mode 100644 index 0000000..c5b83df --- /dev/null +++ b/.github/workflows/Lint-and-test.yml @@ -0,0 +1,38 @@ +name: Lint-and-test +on: [push, workflow_call] +jobs: + tests: + runs-on: ${{ matrix.os }} + strategy: + matrix: + os: [ "ubuntu-latest", "windows-latest" ] + version: ['3.12'] + fail-fast: false + steps: + - uses: actions/checkout@v6 + - uses: actions/setup-python@v6 + with: + python-version: ${{ matrix.version }} + - name: install requirements + run: pip install -e .[dev] + - name: run ruff check + run: python -m ruff check + - name: run ruff format + run: python -m ruff format --check + - name: run pyright + run: python -m pyright + - name: run pytest + run: python -m pytest + results: + if: ${{ always() }} + runs-on: ubuntu-latest + name: Final Results + needs: [tests] + steps: + - run: exit 1 + # see https://stackoverflow.com/a/67532120/4907315 + if: >- + ${{ + contains(needs.*.result, 'failure') + || contains(needs.*.result, 'cancelled') + }} diff --git a/.github/workflows/check_pr_has_label.yml b/.github/workflows/check_pr_has_label.yml new file mode 100644 index 0000000..2a932a2 --- /dev/null +++ b/.github/workflows/check_pr_has_label.yml @@ -0,0 +1,24 @@ +name: Check PR has release labels +on: + pull_request: + types: + - opened + - reopened + - synchronize + - labeled + - unlabeled + +jobs: + has_label: + name: Check PR has release labels + runs-on: ubuntu-latest + steps: + - run: | + echo "PR does not have a release label." + exit 1 + if: | + !contains(github.event.pull_request.labels.*.name, 'Semver-Patch') && + !contains(github.event.pull_request.labels.*.name, 'Semver-Major') && + !contains(github.event.pull_request.labels.*.name, 'Semver-Minor') && + !contains(github.event.pull_request.labels.*.name, 'Semver-Docs') && + !contains(github.event.pull_request.labels.*.name, 'Semver-Ignore') diff --git a/.github/workflows/dependabot-prs.yml b/.github/workflows/dependabot-prs.yml new file mode 100644 index 0000000..219103b --- /dev/null +++ b/.github/workflows/dependabot-prs.yml @@ -0,0 +1,19 @@ +name: Add dependabot PRs to flash reviews +on: + pull_request: + types: + - opened + - reopened + - labeled + +jobs: + add_flash_review: + name: Add dependabot PRs to flash reviews + runs-on: ubuntu-latest + steps: + - uses: actions/add-to-project@v1.0.2 + with: + project-url: https://github.com/orgs/ISISComputingGroup/projects/17 + github-token: ${{ secrets.PROJECT_TOKEN }} + labeled: dependencies + label-operator: OR diff --git a/.github/workflows/documentation.yml b/.github/workflows/documentation.yml new file mode 100644 index 0000000..4235acb --- /dev/null +++ b/.github/workflows/documentation.yml @@ -0,0 +1,21 @@ +name: sphinx + +on: [push, pull_request, workflow_call] + +jobs: + spellcheck: + runs-on: "windows-latest" + steps: + - uses: actions/checkout@v6 + with: + fetch-depth: 0 + - uses: actions/setup-python@v6 + with: + python-version: "3.13" + - name: install requirements + run: pip install -e .[dev] + - name: run spellcheck + run: sphinx-build -E -a -W --keep-going -b spelling doc _build + call_sphinx_builder: + uses: ISISComputingGroup/reusable-workflows/.github/workflows/sphinx.yml@main + secrets: inherit diff --git a/.github/workflows/lint-and-test-nightly.yml b/.github/workflows/lint-and-test-nightly.yml new file mode 100644 index 0000000..5119ad4 --- /dev/null +++ b/.github/workflows/lint-and-test-nightly.yml @@ -0,0 +1,9 @@ +name: lint-and-test-nightly +on: + schedule: + - cron: "0 0 * * *" + workflow_dispatch: + +jobs: + lint-and-test-nightly: + uses: ./.github/workflows/Lint-and-test.yml diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml new file mode 100644 index 0000000..74b53bf --- /dev/null +++ b/.github/workflows/release.yml @@ -0,0 +1,88 @@ +name: Publish Python distribution to PyPI +on: push +jobs: + lint-and-test: + if: github.ref_type == 'tag' + name: Run linter and tests + uses: ./.github/workflows/Lint-and-test.yml + build: + needs: lint-and-test + if: github.ref_type == 'tag' + name: build distribution + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v6 + - name: Set up Python + uses: actions/setup-python@v6 + with: + python-version: "3.13" + - name: Install pypa/build + run: >- + python3 -m + pip install + build + --user + - name: Build a binary wheel and a source tarball + run: python3 -m build + - name: Store the distribution packages + uses: actions/upload-artifact@v6 + with: + name: python-package-distributions + path: dist/ + publish-to-pypi: + name: >- + Publish Python distribution to PyPI + if: github.ref_type == 'tag' + needs: [lint-and-test, build] + runs-on: ubuntu-latest + environment: + name: release + url: https://pypi.org/p/fastcs-secop + permissions: + id-token: write # IMPORTANT: mandatory for trusted publishing + steps: + - name: Download all the dists + uses: actions/download-artifact@v7 + with: + name: python-package-distributions + path: dist/ + - name: Publish distribution to PyPI + uses: pypa/gh-action-pypi-publish@release/v1 + github-release: + name: >- + Sign the Python distribution with Sigstore + and upload them to GitHub Release + needs: [lint-and-test, build, publish-to-pypi] + runs-on: ubuntu-latest + permissions: + contents: write # IMPORTANT: mandatory for making GitHub Releases + id-token: write # IMPORTANT: mandatory for sigstore + steps: + - name: Download all the dists + uses: actions/download-artifact@v7 + with: + name: python-package-distributions + path: dist/ + - name: Sign the dists with Sigstore + uses: sigstore/gh-action-sigstore-python@v3.2.0 + with: + inputs: >- + ./dist/*.tar.gz + ./dist/*.whl + - name: Create GitHub Release + uses: softprops/action-gh-release@v2 + with: + generate_release_notes: true + env: + GITHUB_TOKEN: ${{ github.token }} + - name: Upload artifact signatures to GitHub Release + env: + GITHUB_TOKEN: ${{ github.token }} + # Upload to GitHub Release using the `gh` CLI. + # `dist/` contains the built packages, and the + # sigstore-produced signatures and certificates. + run: >- + gh release upload + '${{ github.ref_name }}' dist/** + --repo '${{ github.repository }}' diff --git a/pyproject.toml b/pyproject.toml index f89289b..44dd0d3 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -33,7 +33,7 @@ classifiers = [ ] dependencies = [ - "fastcs[epics]", + "fastcs[epics] @ git+https://github.com/Tom-Willemsen/FastCS@guard_methods_that_dont_work_on_windows", ] [project.optional-dependencies] @@ -81,7 +81,7 @@ exclude_lines = [ directory = "coverage_html_report" [tool.pyright] -include = ["src", "tests"] +include = ["src"] reportConstantRedefinition = true reportDeprecated = true reportInconsistentConstructor = true diff --git a/ruff.toml b/ruff.toml new file mode 100644 index 0000000..4f22e05 --- /dev/null +++ b/ruff.toml @@ -0,0 +1,53 @@ +line-length = 100 +indent-width = 4 + +[lint] +preview = true +extend-select = [ + "N", # pep8-naming + "D", # pydocstyle + "I", # isort (for imports) + "E501", # Line too long ({width} > {limit}) + "E", # Pycodestyle errors + "W", # Pycodestyle warnings + "F", # Pyflakes + "PL", # Pylint + "B", # Flake8-bugbear + "PIE", # Flake8-pie + "ANN", # Annotations + "ASYNC", # Asyncio-specific checks + "NPY", # Numpy-specific rules + "RUF", # Ruff-specific checks, include some useful asyncio rules + "FURB", # Rules from refurb + "ERA", # Commented-out code + "PT", # Pytest-specific rules + "LOG", # Logging-specific rules + "G", # Logging-specific rules + "UP", # Pyupgrade + "SLF", # Private-member usage + "PERF", # Performance-related rules +] +ignore = [ + "D406", # Section name should end with a newline ("{name}") + "D407", # Missing dashed underline after section ("{name}") + "D213", # Incompatible with D212 + "D203", # Incompatible with D211 + "PLR6301" # Too noisy +] +[lint.per-file-ignores] +"tests/*" = [ + "N802", # Allow test names to be long / not pep8 + "D", # Don't require method documentation for test methods + "ANN", # Don't require tests to use type annotations + "PLR2004", # Allow magic numbers in tests + "PLR0915", # Allow complex tests + "PLR0914", # Allow complex tests + "PLC2701", # Allow tests to import "private" things + "SLF001", # Allow tests to use "private" things +] +"doc/conf.py" = [ + "D100" +] + +[lint.pylint] +max-args = 8 diff --git a/src/fastcs_secop/__init__.py b/src/fastcs_secop/__init__.py index 361c1e5..b0a3d92 100644 --- a/src/fastcs_secop/__init__.py +++ b/src/fastcs_secop/__init__.py @@ -1,3 +1,5 @@ +"""SECoP support using FastCS.""" + import logging from logging import getLogger @@ -14,12 +16,12 @@ __all__ = [ "SecopController", - "SecopModuleController", "SecopError", + "SecopModuleController", ] -if __name__ == "__main__": +if __name__ == "__main__": # pragma: no cover configure_logging(level=LogLevel.DEBUG) logging.basicConfig(level=LogLevel.DEBUG) diff --git a/src/fastcs_secop/_controllers.py b/src/fastcs_secop/_controllers.py index 93a7953..34f3be8 100644 --- a/src/fastcs_secop/_controllers.py +++ b/src/fastcs_secop/_controllers.py @@ -20,9 +20,7 @@ def format_string_to_prec(fmt_str: str | None) -> int | None: - """ - Convert a SECoP format-string specifier to a precision. - """ + """Convert a SECoP format-string specifier to a precision.""" if fmt_str is None: return None @@ -32,13 +30,13 @@ def format_string_to_prec(fmt_str: str | None) -> int | None: return None -def secop_dtype_to_numpy_dtype(secop_dtype: str) -> str: +def secop_dtype_to_numpy_dtype(secop_dtype: str) -> npt.DTypeLike: if secop_dtype == "double": - return "float64" + return np.float64 elif secop_dtype == "int": - return "int32" + return np.int32 elif secop_dtype == "bool": - return "int32" + return np.bool_ else: raise SecopError(f"Cannot handle SECoP dtype '{secop_dtype}' within array/struct/tuple") @@ -68,7 +66,7 @@ async def initialise(self) -> None: if secop_dtype == "command": # TODO: handle commands pass - elif secop_dtype in ["double", "scaled"]: + elif secop_dtype in {"double", "scaled"}: if min_val is not None and scale is not None: min_val *= scale if max_val is not None and scale is not None: @@ -88,7 +86,7 @@ async def initialise(self) -> None: accessible_name=parameter_name, update_period=1.0, decode=lambda x, s=scale: x * s if s is not None else x, - encode=lambda x, s=scale: int(round(x / s)) if s is not None else x, + encode=lambda x, s=scale: round(x / s) if s is not None else x, ), description=parameter.get("description", ""), ), @@ -169,14 +167,11 @@ async def initialise(self) -> None: ) elif secop_dtype == "array": inner_dtype = datainfo["members"]["type"] - if inner_dtype not in ["double", "int", "bool"]: - raise SecopError(f"Cannot handle inner dtype {inner_dtype} in array.") - np_inner_dtype = secop_dtype_to_numpy_dtype(inner_dtype) def decode( x: list[int | float | bool], t: npt.DTypeLike = np_inner_dtype - ) -> npt.NDArray: + ) -> npt.NDArray[np.int32 | np.float64 | np.bool_]: return np.array(x, dtype=t) self.add_attribute( @@ -223,20 +218,20 @@ async def ping(self) -> None: logger.info("Reconnect failed.") async def check_idn(self) -> None: - """ - Checks that the response to *IDN? indicates this is a SECoP device. + """Check that the response to *IDN? indicates this is a SECoP device. Raises: ValueError: if device is not a SECoP device. + """ identification = await self._connection.send_query("*IDN?\n") identification = identification.strip() - manufacturer, product, draft_date, version = identification.split(",") - if manufacturer not in [ + manufacturer, product, _, _ = identification.split(",") + if manufacturer not in { "ISSE&SINE2020", # SECOP 1.x "ISSE", # SECOP 2.x - ]: + }: raise SecopError( f"Device responded to '*IDN?' with bad manufacturer string '{manufacturer}'. " f"Not a SECoP device?" diff --git a/src/fastcs_secop/_io.py b/src/fastcs_secop/_io.py index 5bc55f2..eda0cc3 100644 --- a/src/fastcs_secop/_io.py +++ b/src/fastcs_secop/_io.py @@ -17,8 +17,8 @@ class SecopAttributeIORef(AttributeIORef): module_name: str = "" accessible_name: str = "" - decode: callable = lambda x: x - encode: callable = lambda x: x + decode: typing.Callable[[typing.Any], typing.Any] = lambda x: x + encode: typing.Callable[[typing.Any], typing.Any] = lambda x: x class SecopAttributeIO(AttributeIO[NumberT, SecopAttributeIORef]): diff --git a/tests/emulators/__init__.py b/tests/emulators/__init__.py index c396168..e69de29 100644 --- a/tests/emulators/__init__.py +++ b/tests/emulators/__init__.py @@ -1 +0,0 @@ -from __future__ import absolute_import diff --git a/tests/emulators/simple_secop/device.py b/tests/emulators/simple_secop/device.py index 41a07ab..7bddb73 100644 --- a/tests/emulators/simple_secop/device.py +++ b/tests/emulators/simple_secop/device.py @@ -1,4 +1,5 @@ import base64 +import math import time import typing from collections import OrderedDict @@ -99,7 +100,7 @@ def __init__(self): extra_datainfo={"maxbytes": 512}, ), "double_array": Parameter( - [1.414, 1.618, 2.718, 3.14159], + [1.414, 1.618, math.e, math.pi], desc="a double array parameter", dtype="array", extra_datainfo={"maxlen": 512, "members": {"type": "double"}}, diff --git a/tests/emulators/simple_secop/interfaces/stream_interface.py b/tests/emulators/simple_secop/interfaces/stream_interface.py index 83d4080..195e9d3 100644 --- a/tests/emulators/simple_secop/interfaces/stream_interface.py +++ b/tests/emulators/simple_secop/interfaces/stream_interface.py @@ -9,7 +9,7 @@ @has_log class SimpleSecopStreamInterface(StreamInterface): - commands = { + commands: typing.ClassVar = { CmdBuilder("idn").escape("*IDN?").optional("\r").eos().build(), CmdBuilder("ping").escape("ping ").any().optional("\r").eos().build(), CmdBuilder("deactivate").escape("deactivate").optional(" .").optional("\r").eos().build(), @@ -39,9 +39,7 @@ class SimpleSecopStreamInterface(StreamInterface): out_terminator = "\n" def handle_error(self, request, error): - err_string = "command was: {}, error was: {}: {}\n".format( - request, error.__class__.__name__, error - ) + err_string = f"command was: {request}, error was: {error.__class__.__name__}: {error}\n" print(err_string) self.log.error(err_string) return err_string diff --git a/tests/test_against_emulator.py b/tests/test_against_emulator.py index 4063060..d0062f7 100644 --- a/tests/test_against_emulator.py +++ b/tests/test_against_emulator.py @@ -1,4 +1,5 @@ import asyncio +import math import os.path import subprocess import sys @@ -27,7 +28,7 @@ def emulator(): "emulators", "simple_secop", "-p", - "stream: {bind_address: localhost, port: 57677}", + "stream: {bind_address: 127.0.0.1, port: 57677}", ], cwd=os.path.dirname(__file__), ) @@ -46,6 +47,15 @@ async def controller(): ) ) + for _ in range(100): + try: + await controller.connect() + break + except Exception: + await asyncio.sleep(0.1) + else: + raise RuntimeError("Could not connect to emulator") + fastcs = FastCS( controller, [], @@ -55,7 +65,7 @@ async def controller(): # Wait for FastCS to have run initialise() & created attributes max_iters = 100 # 10 seconds - for i in range(max_iters): + for _ in range(max_iters): if controller.sub_controllers: break await asyncio.sleep(0.1) @@ -69,12 +79,12 @@ async def controller(): await fastcs_task -async def test_sub_controllers_created(controller): +def test_sub_controllers_created(controller): assert "one_of_everything" in controller.sub_controllers @pytest.mark.parametrize( - "param,expected_initial_value", + ("param", "expected_initial_value"), [ ("double", 1.2345), ("scaled", 42 * 47), @@ -98,12 +108,12 @@ async def test_attributes_created_for_enum_datatype(controller): @pytest.mark.parametrize( - "param,expected_initial_value", + ("param", "expected_initial_value"), [ ("blob", np.array([c for c in b"a blob of binary data"], dtype=np.uint8)), ("int_array", np.array([1, 1, 2, 3, 5, 8, 13], dtype=np.int64)), ("bool_array", np.array([1, 1, 0, 1, 0, 0, 1, 1], dtype=np.bool_)), - ("double_array", np.array([1.414, 1.618, 2.718, 3.14159], dtype=np.float64)), + ("double_array", np.array([1.414, 1.618, math.e, math.pi], dtype=np.float64)), ], ) async def test_attributes_created_for_array_datatype(controller, param, expected_initial_value): diff --git a/tests/test_controller.py b/tests/test_controller.py index 91de28f..deafed3 100644 --- a/tests/test_controller.py +++ b/tests/test_controller.py @@ -1,9 +1,38 @@ -from fastcs_secop._controllers import format_string_to_prec +import numpy as np +import pytest +from fastcs_secop import SecopError +from fastcs_secop._controllers import format_string_to_prec, secop_dtype_to_numpy_dtype -def test_format_string_to_prec(): - assert format_string_to_prec("%.1f") == 1 - assert format_string_to_prec("%.99f") == 99 - assert format_string_to_prec("%.5g") is None - assert format_string_to_prec("%.5e") is None - assert format_string_to_prec(None) is None + +@pytest.mark.parametrize( + ("secop_fmt", "prec"), + [ + ("%.1f", 1), + ("%.99f", 99), + ("%.5g", None), + ("%.5e", None), + (None, None), + ], +) +def test_format_string_to_prec(secop_fmt, prec): + assert format_string_to_prec(secop_fmt) == prec + + +@pytest.mark.parametrize( + ("secop_dtype", "np_dtype"), + [ + ("int", np.int32), + ("double", np.float64), + ("bool", np.bool_), + ], +) +def test_secop_dtype_to_numpy_dtype(secop_dtype, np_dtype): + assert secop_dtype_to_numpy_dtype(secop_dtype) == np_dtype + + +def test_invalid_secop_dtype_to_numpy_dtype(): + with pytest.raises( + SecopError, match=r"Cannot handle SECoP dtype 'array' within array/struct/tuple" + ): + secop_dtype_to_numpy_dtype("array") From 2935e5b1f7659c5e05aced0f11ee9a018fe1ca06 Mon Sep 17 00:00:00 2001 From: Tom Willemsen Date: Thu, 25 Dec 2025 08:07:53 +0000 Subject: [PATCH 08/19] doc --- LICENSE | 28 ++++++++ doc/_api.rst | 11 +++ doc/_static/css/custom.css | 7 ++ doc/_templates/custom-module-template.rst | 40 +++++++++++ doc/conf.py | 87 +++++++++++++++++++++++ doc/index.md | 11 +++ doc/spelling_wordlist.txt | 0 pyproject.toml | 7 +- src/fastcs_secop/__init__.py | 6 +- src/fastcs_secop/_controllers.py | 58 +++++++++++++-- src/fastcs_secop/_util.py | 2 +- tests/test_against_emulator.py | 2 +- tests/test_controller.py | 41 ++++++++++- 13 files changed, 286 insertions(+), 14 deletions(-) create mode 100644 LICENSE create mode 100644 doc/_api.rst create mode 100644 doc/_static/css/custom.css create mode 100644 doc/_templates/custom-module-template.rst create mode 100644 doc/conf.py create mode 100644 doc/index.md create mode 100644 doc/spelling_wordlist.txt diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..b44c480 --- /dev/null +++ b/LICENSE @@ -0,0 +1,28 @@ +BSD 3-Clause License + +Copyright (c) 2026, ISIS Experiment Controls Computing + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + +1. Redistributions of source code must retain the above copyright notice, this + list of conditions and the following disclaimer. + +2. Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimer in the documentation + and/or other materials provided with the distribution. + +3. Neither the name of the copyright holder nor the names of its + contributors may be used to endorse or promote products derived from + this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/doc/_api.rst b/doc/_api.rst new file mode 100644 index 0000000..0378f50 --- /dev/null +++ b/doc/_api.rst @@ -0,0 +1,11 @@ +:orphan: + +API +=== + +.. autosummary:: + :toctree: generated + :template: custom-module-template.rst + :recursive: + + fastcs_secop diff --git a/doc/_static/css/custom.css b/doc/_static/css/custom.css new file mode 100644 index 0000000..1a1c10a --- /dev/null +++ b/doc/_static/css/custom.css @@ -0,0 +1,7 @@ +a:hover { + color: #d2b48c; +} + +.wy-menu-vertical p.caption { + color: #d2b48c; +} diff --git a/doc/_templates/custom-module-template.rst b/doc/_templates/custom-module-template.rst new file mode 100644 index 0000000..f483c41 --- /dev/null +++ b/doc/_templates/custom-module-template.rst @@ -0,0 +1,40 @@ + + +{{ ('``' + fullname + '``') | underline }} + +{%- set filtered_members = [] %} +{%- for item in members %} + {%- if item in functions + classes + exceptions + attributes %} + {% set _ = filtered_members.append(item) %} + {%- endif %} +{%- endfor %} + +.. automodule:: {{ fullname }} + :members: + :show-inheritance: + + {% block modules %} + {% if modules %} + .. rubric:: Submodules + + .. autosummary:: + :toctree: + :template: custom-module-template.rst + :recursive: + {% for item in modules %} + {{ item }} + {%- endfor %} + {% endif %} + {% endblock %} + + {% block members %} + {% if filtered_members %} + .. rubric:: Members + + .. autosummary:: + :nosignatures: + {% for item in filtered_members %} + {{ item }} + {%- endfor %} + {% endif %} + {% endblock %} \ No newline at end of file diff --git a/doc/conf.py b/doc/conf.py new file mode 100644 index 0000000..81573b5 --- /dev/null +++ b/doc/conf.py @@ -0,0 +1,87 @@ +# Configuration file for the Sphinx documentation builder. +# +# For the full list of built-in configuration values, see the documentation: +# https://www.sphinx-doc.org/en/master/usage/configuration.html + +# -- Project information ----------------------------------------------------- +# https://www.sphinx-doc.org/en/master/usage/configuration.html#project-information + +import os +import sys + +import fastcs_secop # noqa: F401 + +sys.path.insert(0, os.path.abspath("../src")) + +project = "fastcs-secop" +copyright = "" +author = "ISIS Experiment Controls" +release = "0.1" + +# -- General configuration --------------------------------------------------- +# https://www.sphinx-doc.org/en/master/usage/configuration.html#general-configuration + +nitpicky = True + +myst_enable_extensions = ["dollarmath", "strikethrough", "colon_fence", "attrs_block"] +suppress_warnings = ["myst.strikethrough"] + +extensions = [ + "myst_parser", + "sphinx.ext.autodoc", + # and making summary tables at the top of API docs + "sphinx.ext.autosummary", + # This can parse google style docstrings + "sphinx.ext.napoleon", + # For linking to external sphinx documentation + "sphinx.ext.intersphinx", + # Add links to source code in API docs + "sphinx.ext.viewcode", + # Mermaid diagrams + "sphinxcontrib.mermaid", +] +mermaid_d3_zoom = True +napoleon_google_docstring = True +napoleon_numpy_docstring = False + +templates_path = ["_templates"] +exclude_patterns = ["_build", "Thumbs.db", ".DS_Store"] + +# -- Options for HTML output ------------------------------------------------- +# https://www.sphinx-doc.org/en/master/usage/configuration.html#options-for-html-output + +html_context = { + "display_github": True, # Integrate GitHub + "github_user": "ISISComputingGroup", # Username + "github_repo": "fastcs-secop", # Repo name + "github_version": "main", # Version + "conf_py_path": "/doc/", # Path in the checkout to the docs root +} + +html_theme = "sphinx_rtd_theme" +html_logo = "logo.svg" +html_theme_options = { + "logo_only": False, + "style_nav_header_background": "#343131", +} +html_favicon = "favicon.svg" +html_static_path = ["_static"] +html_css_files = [ + "css/custom.css", +] + +autoclass_content = "init" +myst_heading_anchors = 7 +autodoc_preserve_defaults = True + +spelling_lang = "en_GB" +spelling_filters = ["enchant.tokenize.MentionFilter"] +spelling_warning = True +spelling_show_suggestions = True +spelling_suggestion_limit = 3 + +intersphinx_mapping = { + "python": ("https://docs.python.org/3", None), + "secop": ("https://sampleenvironment.github.io/secop-site/", None), + "fastcs": ("https://diamondlightsource.github.io/FastCS/main/", None), +} diff --git a/doc/index.md b/doc/index.md new file mode 100644 index 0000000..a15fb01 --- /dev/null +++ b/doc/index.md @@ -0,0 +1,11 @@ +# `fastcs-secop` documentation + +The {py:obj}`fastcs_secop` library implements support for the {external+secop:doc}`SECoP protocol ` +using {external+fastcs:doc}`FastCS `. + +```{toctree} +:titlesonly: +:caption: Reference + +_api +``` diff --git a/doc/spelling_wordlist.txt b/doc/spelling_wordlist.txt new file mode 100644 index 0000000..e69de29 diff --git a/pyproject.toml b/pyproject.toml index 44dd0d3..2e53434 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -63,10 +63,16 @@ dev = [ testpaths = "tests" asyncio_mode = "auto" addopts = "--cov --cov-report=html -vv" +filterwarnings = [ + 'ignore::tango.PyTangoUserWarning', +] [tool.coverage.run] branch = true source = ["src"] +omit = [ + "version.py", # Autogenerated +] [tool.coverage.report] fail_under = 100 @@ -99,4 +105,3 @@ reportUntypedFunctionDecorator = true version_file = "src/fastcs_secop/version.py" [tool.build_sphinx] - diff --git a/src/fastcs_secop/__init__.py b/src/fastcs_secop/__init__.py index b0a3d92..a5eb067 100644 --- a/src/fastcs_secop/__init__.py +++ b/src/fastcs_secop/__init__.py @@ -14,11 +14,7 @@ logger = getLogger(__name__) -__all__ = [ - "SecopController", - "SecopError", - "SecopModuleController", -] +__all__ = ["SecopController", "SecopError", "SecopModuleController"] if __name__ == "__main__": # pragma: no cover diff --git a/src/fastcs_secop/_controllers.py b/src/fastcs_secop/_controllers.py index 34f3be8..ea53269 100644 --- a/src/fastcs_secop/_controllers.py +++ b/src/fastcs_secop/_controllers.py @@ -49,6 +49,19 @@ def __init__( module_name: str, module: dict[str, typing.Any], ) -> None: + """FastCS controller for a SECoP module. + + Instances of this class are added as subcontrollers by + :py:obj:`SecopController`. + + Args: + connection: The connection to use. + module_name: The name of the SECoP module. + module: A deserialised description, in the + :external+secop:doc:`SECoP over-the-wire format `, + of this module. + + """ self._io = SecopAttributeIO(connection=connection) self._module_name = module_name self._module = module @@ -191,6 +204,13 @@ def decode( class SecopController(Controller): def __init__(self, settings: IPConnectionSettings) -> None: + """FastCS Controller for a SECoP node. + + Args: + settings: The communication settings (e.g. IP address, port) at which + the SECoP node is reachable. + + """ self._ip_settings = settings self._connection = IPConnection() @@ -199,6 +219,13 @@ def __init__(self, settings: IPConnectionSettings) -> None: async def connect(self) -> None: await self._connection.connect(self._ip_settings) + async def deactivate(self) -> None: + """Turn off asynchronous SECoP communication. + + See :external+secop:doc:`specification/messages/activation` for details. + """ + await self._connection.send_query("deactivate\n") + @scan(15.0) async def ping(self) -> None: """Ping the SECoP device, to check connection is still open. @@ -213,15 +240,19 @@ async def ping(self) -> None: logger.info("Detected connection loss, attempting reconnect.") try: await self.connect() + await self.deactivate() logger.info("Reconnect successful.") except Exception: logger.info("Reconnect failed.") async def check_idn(self) -> None: - """Check that the response to *IDN? indicates this is a SECoP device. + """Verify that the device is a SECoP device. + + This is checked using the SECoP + :external+secop:doc:`identification message `. Raises: - ValueError: if device is not a SECoP device. + SecopError: if the device is not a SECoP device. """ identification = await self._connection.send_query("*IDN?\n") @@ -246,11 +277,28 @@ async def check_idn(self) -> None: print(f"Connected to SECoP device with IDN='{identification}'.") async def initialise(self) -> None: + """Set up FastCS for this SECoP node. + + This introspects the + :external+secop:doc:`description ` + of the SECoP device to determine the names and contents of the modules + in this SECoP node. + + A subcontroller of type :py:obj:`SecopModuleController` is added for + each discovered module. + + This controller attempts to periodically reconnect to the device if the + connection was closed, and disables asynchronous messages on instantiation. + + Raises: + SecopError: if the device is not a SECoP device, if a reply in an + unexpected format is received, or the SECoP node's configuration + cannot be handled by :py:obj:`fastcs_secop`. + + """ await self.connect() await self.check_idn() - - # Turn off asynchronous replies. - await self._connection.send_query("deactivate\n") + await self.deactivate() descriptor = await self._connection.send_query("describe\n") if not descriptor.startswith("describing . "): diff --git a/src/fastcs_secop/_util.py b/src/fastcs_secop/_util.py index 73f935b..e3b9ac8 100644 --- a/src/fastcs_secop/_util.py +++ b/src/fastcs_secop/_util.py @@ -1,2 +1,2 @@ class SecopError(Exception): - pass + """Error raised to identify a SECoP protocol or configuration problem.""" diff --git a/tests/test_against_emulator.py b/tests/test_against_emulator.py index d0062f7..9c4ede7 100644 --- a/tests/test_against_emulator.py +++ b/tests/test_against_emulator.py @@ -54,7 +54,7 @@ async def controller(): except Exception: await asyncio.sleep(0.1) else: - raise RuntimeError("Could not connect to emulator") + raise RuntimeError("Could not connect to emulator within 10s") fastcs = FastCS( controller, diff --git a/tests/test_controller.py b/tests/test_controller.py index deafed3..ea75b8e 100644 --- a/tests/test_controller.py +++ b/tests/test_controller.py @@ -1,8 +1,26 @@ +from unittest.mock import AsyncMock, patch + import numpy as np import pytest +from fastcs.connections import IPConnectionSettings from fastcs_secop import SecopError -from fastcs_secop._controllers import format_string_to_prec, secop_dtype_to_numpy_dtype +from fastcs_secop._controllers import ( + SecopController, + format_string_to_prec, + secop_dtype_to_numpy_dtype, +) + + +@pytest.fixture +def controller(): + controller = SecopController( + settings=IPConnectionSettings( + ip="127.0.0.1", + port=65535, + ) + ) + return controller @pytest.mark.parametrize( @@ -36,3 +54,24 @@ def test_invalid_secop_dtype_to_numpy_dtype(): SecopError, match=r"Cannot handle SECoP dtype 'array' within array/struct/tuple" ): secop_dtype_to_numpy_dtype("array") + + +async def test_ping_happy_path(controller): + with patch.object(controller._connection, "send_query", AsyncMock(return_value="pong")): + controller.connect = AsyncMock() + await controller.ping() + controller.connect.assert_not_awaited() + + +async def test_ping_raises_disconnected_error(controller): + with patch.object(controller._connection, "send_query", AsyncMock(side_effect=ConnectionError)): + controller.connect = AsyncMock() + await controller.ping() + controller.connect.assert_awaited() + + +async def test_ping_raises_disconnected_error_and_reconnect_fails(controller): + with patch.object(controller._connection, "send_query", AsyncMock(side_effect=ConnectionError)): + controller.connect = AsyncMock(side_effect=ConnectionError) + await controller.ping() + controller.connect.assert_awaited() From 71993bcfd7e0983207ae79adb1c3cdd10c818164 Mon Sep 17 00:00:00 2001 From: Tom Willemsen Date: Thu, 25 Dec 2025 09:12:55 +0000 Subject: [PATCH 09/19] Support tuples --- src/fastcs_secop/__init__.py | 2 +- src/fastcs_secop/_controllers.py | 50 +++++++++++---- tests/emulators/simple_secop/device.py | 6 ++ tests/test_against_emulator.py | 86 ++++++++++++++------------ tests/test_controller.py | 2 +- 5 files changed, 94 insertions(+), 52 deletions(-) diff --git a/src/fastcs_secop/__init__.py b/src/fastcs_secop/__init__.py index a5eb067..0f00aff 100644 --- a/src/fastcs_secop/__init__.py +++ b/src/fastcs_secop/__init__.py @@ -28,6 +28,6 @@ fastcs = FastCS( SecopController(settings=IPConnectionSettings(ip="127.0.0.1", port=57677)), - [epics_ca], + [epics_pva], ) fastcs.run(interactive=False) diff --git a/src/fastcs_secop/_controllers.py b/src/fastcs_secop/_controllers.py index ea53269..a93a799 100644 --- a/src/fastcs_secop/_controllers.py +++ b/src/fastcs_secop/_controllers.py @@ -7,10 +7,10 @@ import numpy as np import numpy.typing as npt -from fastcs.attributes import AttrRW +from fastcs.attributes import AttrR, AttrRW from fastcs.connections import IPConnection, IPConnectionSettings from fastcs.controllers import Controller -from fastcs.datatypes import Bool, Enum, Float, Int, String, Waveform +from fastcs.datatypes import Bool, Enum, Float, Int, String, Table, Waveform from fastcs.methods import scan from fastcs_secop._io import SecopAttributeIO, SecopAttributeIORef @@ -36,7 +36,7 @@ def secop_dtype_to_numpy_dtype(secop_dtype: str) -> npt.DTypeLike: elif secop_dtype == "int": return np.int32 elif secop_dtype == "bool": - return np.bool_ + return np.uint8 else: raise SecopError(f"Cannot handle SECoP dtype '{secop_dtype}' within array/struct/tuple") @@ -62,10 +62,9 @@ def __init__( of this module. """ - self._io = SecopAttributeIO(connection=connection) self._module_name = module_name self._module = module - super().__init__(ios=[self._io]) + super().__init__(ios=[SecopAttributeIO(connection=connection)]) async def initialise(self) -> None: for parameter_name, parameter in self._module["accessibles"].items(): @@ -76,6 +75,8 @@ async def initialise(self) -> None: max_val = datainfo.get("max") scale = datainfo.get("scale") + attr_cls = AttrR if parameter.get("readonly", False) else AttrRW + if secop_dtype == "command": # TODO: handle commands pass @@ -87,7 +88,7 @@ async def initialise(self) -> None: self.add_attribute( parameter_name, - AttrRW( + attr_cls( Float( units=datainfo.get("unit", None), min_alarm=min_val, @@ -107,7 +108,7 @@ async def initialise(self) -> None: elif secop_dtype == "int": self.add_attribute( parameter_name, - AttrRW( + attr_cls( Int( units=datainfo.get("unit", None), min_alarm=min_val, @@ -124,7 +125,7 @@ async def initialise(self) -> None: elif secop_dtype == "bool": self.add_attribute( parameter_name, - AttrRW( + attr_cls( Bool(), io_ref=SecopAttributeIORef( module_name=self._module_name, @@ -140,7 +141,7 @@ async def initialise(self) -> None: self.add_attribute( parameter_name, - AttrRW( + attr_cls( Enum(enum_type), io_ref=SecopAttributeIORef( module_name=self._module_name, @@ -153,7 +154,7 @@ async def initialise(self) -> None: elif secop_dtype == "string": self.add_attribute( parameter_name, - AttrRW( + attr_cls( String(), io_ref=SecopAttributeIORef( module_name=self._module_name, @@ -166,7 +167,7 @@ async def initialise(self) -> None: elif secop_dtype == "blob": self.add_attribute( parameter_name, - AttrRW( + attr_cls( Waveform(np.uint8, shape=(datainfo["maxbytes"],)), io_ref=SecopAttributeIORef( module_name=self._module_name, @@ -189,7 +190,7 @@ def decode( self.add_attribute( parameter_name, - AttrRW( + attr_cls( Waveform(np_inner_dtype, shape=(datainfo["maxlen"],)), io_ref=SecopAttributeIORef( module_name=self._module_name, @@ -200,6 +201,31 @@ def decode( description=parameter.get("description", ""), ), ) + elif secop_dtype == "tuple": + secop_dtypes = [t["type"] for t in datainfo["members"]] + np_dtypes = [secop_dtype_to_numpy_dtype(t) for t in secop_dtypes] + names = [f"e{n}" for n in range(len(datainfo["members"]))] + + structured_dtype = list(zip(names, np_dtypes, strict=True)) + + def decode( + val: list[int | float | bool], t: npt.DTypeLike = structured_dtype + ) -> npt.NDArray[structured_dtype]: + return np.array([tuple(val)], dtype=t) + + self.add_attribute( + parameter_name, + attr_cls( + Table(structured_dtype), + io_ref=SecopAttributeIORef( + module_name=self._module_name, + accessible_name=parameter_name, + update_period=1.0, + decode=decode, + ), + description=parameter.get("description", ""), + ), + ) class SecopController(Controller): diff --git a/tests/emulators/simple_secop/device.py b/tests/emulators/simple_secop/device.py index 7bddb73..69d35aa 100644 --- a/tests/emulators/simple_secop/device.py +++ b/tests/emulators/simple_secop/device.py @@ -117,6 +117,12 @@ def __init__(self): dtype="array", extra_datainfo={"maxlen": 512, "members": {"type": "bool"}}, ), + "tuple": Parameter( + [1, 5.678, True], + desc="a tuple of int, float, bool", + dtype="tuple", + extra_datainfo={"members": [{"type": "int"}, {"type": "double"}, {"type": "bool"}]}, + ), } self.description = "a module with one accessible of each possible dtype" diff --git a/tests/test_against_emulator.py b/tests/test_against_emulator.py index 9c4ede7..05ae2cd 100644 --- a/tests/test_against_emulator.py +++ b/tests/test_against_emulator.py @@ -17,7 +17,7 @@ configure_logging(level=LogLevel.TRACE) -@pytest.fixture(autouse=True) +@pytest.fixture(autouse=True, scope="class") def emulator(): proc = subprocess.Popen( [ @@ -79,45 +79,55 @@ async def controller(): await fastcs_task -def test_sub_controllers_created(controller): - assert "one_of_everything" in controller.sub_controllers +class TestInitialState: + def test_sub_controllers_created(self, controller): + assert "one_of_everything" in controller.sub_controllers - -@pytest.mark.parametrize( - ("param", "expected_initial_value"), - [ - ("double", 1.2345), - ("scaled", 42 * 47), - ("int", 73), - ("bool", True), - ("string", "hello"), - ], -) -async def test_attributes_created_for_simple_datatype(controller, param, expected_initial_value): - attr: AttrR = typing.cast( - AttrR, controller.sub_controllers["one_of_everything"].attributes[param] + @pytest.mark.parametrize( + ("param", "expected_initial_value"), + [ + ("double", 1.2345), + ("scaled", 42 * 47), + ("int", 73), + ("bool", True), + ("string", "hello"), + ], ) - await attr.wait_for_value(expected_initial_value, timeout=2) + async def test_attributes_created_for_simple_datatype( + self, controller, param, expected_initial_value + ): + attr: AttrR = typing.cast( + AttrR, controller.sub_controllers["one_of_everything"].attributes[param] + ) + await attr.wait_for_value(expected_initial_value, timeout=2) + async def test_attributes_created_for_enum_datatype(self, controller): + attr: AttrR = typing.cast( + AttrR, controller.sub_controllers["one_of_everything"].attributes["enum"] + ) + await attr.wait_for_predicate(lambda v: v.name == "three", timeout=2) -async def test_attributes_created_for_enum_datatype(controller): - attr: AttrR = typing.cast( - AttrR, controller.sub_controllers["one_of_everything"].attributes["enum"] - ) - await attr.wait_for_predicate(lambda v: v.name == "three", timeout=2) - - -@pytest.mark.parametrize( - ("param", "expected_initial_value"), - [ - ("blob", np.array([c for c in b"a blob of binary data"], dtype=np.uint8)), - ("int_array", np.array([1, 1, 2, 3, 5, 8, 13], dtype=np.int64)), - ("bool_array", np.array([1, 1, 0, 1, 0, 0, 1, 1], dtype=np.bool_)), - ("double_array", np.array([1.414, 1.618, math.e, math.pi], dtype=np.float64)), - ], -) -async def test_attributes_created_for_array_datatype(controller, param, expected_initial_value): - attr: AttrR = typing.cast( - AttrR, controller.sub_controllers["one_of_everything"].attributes[param] + @pytest.mark.parametrize( + ("param", "expected_initial_value"), + [ + ("blob", np.array([c for c in b"a blob of binary data"], dtype=np.uint8)), + ("int_array", np.array([1, 1, 2, 3, 5, 8, 13], dtype=np.int32)), + ("bool_array", np.array([1, 1, 0, 1, 0, 0, 1, 1], dtype=np.uint8)), + ("double_array", np.array([1.414, 1.618, math.e, math.pi], dtype=np.float64)), + ( + "tuple", + np.array( + [(1, 5.678, 1)], dtype=[("e0", np.int32), ("e1", np.float64), ("e2", np.uint8)] + ), + ), + ], ) - await attr.wait_for_predicate(lambda v: np.array_equal(v, expected_initial_value), timeout=2) + async def test_attributes_created_for_array_datatype( + self, controller, param, expected_initial_value + ): + attr: AttrR = typing.cast( + AttrR, controller.sub_controllers["one_of_everything"].attributes[param] + ) + await attr.wait_for_predicate( + lambda v: np.array_equal(v, expected_initial_value), timeout=2 + ) diff --git a/tests/test_controller.py b/tests/test_controller.py index ea75b8e..dcfe979 100644 --- a/tests/test_controller.py +++ b/tests/test_controller.py @@ -42,7 +42,7 @@ def test_format_string_to_prec(secop_fmt, prec): [ ("int", np.int32), ("double", np.float64), - ("bool", np.bool_), + ("bool", np.uint8), ], ) def test_secop_dtype_to_numpy_dtype(secop_dtype, np_dtype): From 789d5ecf4a2c1fc918d09cd9edd7f54fb98a284c Mon Sep 17 00:00:00 2001 From: Tom Willemsen Date: Fri, 26 Dec 2025 10:34:29 +0000 Subject: [PATCH 10/19] Support structs --- .github/workflows/documentation.yml | 34 +++- doc/conf.py | 9 +- doc/spelling_wordlist.txt | 6 + pyproject.toml | 3 +- ruff.toml | 2 +- src/fastcs_secop/__init__.py | 4 +- src/fastcs_secop/_io.py | 70 ------- src/fastcs_secop/_util.py | 43 ++++ .../{_controllers.py => controllers.py} | 131 +++++------- src/fastcs_secop/io.py | 186 ++++++++++++++++++ tests/emulators/simple_secop/device.py | 12 ++ tests/test_against_emulator.py | 9 +- tests/test_controller.py | 2 +- 13 files changed, 337 insertions(+), 174 deletions(-) delete mode 100644 src/fastcs_secop/_io.py rename src/fastcs_secop/{_controllers.py => controllers.py} (70%) create mode 100644 src/fastcs_secop/io.py diff --git a/.github/workflows/documentation.yml b/.github/workflows/documentation.yml index 4235acb..3dbc893 100644 --- a/.github/workflows/documentation.yml +++ b/.github/workflows/documentation.yml @@ -3,19 +3,35 @@ name: sphinx on: [push, pull_request, workflow_call] jobs: - spellcheck: - runs-on: "windows-latest" + docs: + runs-on: windows-latest steps: - uses: actions/checkout@v6 with: fetch-depth: 0 - - uses: actions/setup-python@v6 + - uses: actions/setup-python@v5 with: - python-version: "3.13" - - name: install requirements - run: pip install -e .[dev] + python-version: '3.12' + - name: Install dependencies + run: | + pip install .[doc] + - name: Sphinx build + run: sphinx-build -E -a -W --keep-going doc _build - name: run spellcheck run: sphinx-build -E -a -W --keep-going -b spelling doc _build - call_sphinx_builder: - uses: ISISComputingGroup/reusable-workflows/.github/workflows/sphinx.yml@main - secrets: inherit + - name: Upload artifact + uses: actions/upload-artifact@v4 + with: + name: documentation + path: | + _build + if-no-files-found: error + retention-days: 7 + - name: Deploy to GitHub Pages + uses: peaceiris/actions-gh-pages@v3 + if: ${{ github.event_name == 'push' && github.ref == 'refs/heads/main' }} + with: + publish_branch: gh-pages + github_token: ${{ secrets.GITHUB_TOKEN }} + publish_dir: _build/ + force_orphan: true diff --git a/doc/conf.py b/doc/conf.py index 81573b5..00cfc59 100644 --- a/doc/conf.py +++ b/doc/conf.py @@ -22,6 +22,12 @@ # https://www.sphinx-doc.org/en/master/usage/configuration.html#general-configuration nitpicky = True +nitpick_ignore_regex = [ + ("py:class", r"^.*\.T$"), + ("py:obj", r"^.*\.T$"), + ("py:class", r"^.*\.T.*_co$"), + ("py:obj", r"^.*\.T.*_co$"), +] myst_enable_extensions = ["dollarmath", "strikethrough", "colon_fence", "attrs_block"] suppress_warnings = ["myst.strikethrough"] @@ -59,12 +65,10 @@ } html_theme = "sphinx_rtd_theme" -html_logo = "logo.svg" html_theme_options = { "logo_only": False, "style_nav_header_background": "#343131", } -html_favicon = "favicon.svg" html_static_path = ["_static"] html_css_files = [ "css/custom.css", @@ -84,4 +88,5 @@ "python": ("https://docs.python.org/3", None), "secop": ("https://sampleenvironment.github.io/secop-site/", None), "fastcs": ("https://diamondlightsource.github.io/FastCS/main/", None), + "numpy": ("https://numpy.org/doc/stable/", None), } diff --git a/doc/spelling_wordlist.txt b/doc/spelling_wordlist.txt index e69de29..81d8ebe 100644 --- a/doc/spelling_wordlist.txt +++ b/doc/spelling_wordlist.txt @@ -0,0 +1,6 @@ +accessible +accessibles +deserialised +SECoP +subcontroller +subcontrollers diff --git a/pyproject.toml b/pyproject.toml index 2e53434..9ddf14a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -43,6 +43,7 @@ doc = [ "myst_parser", "sphinx-autobuild", "sphinxcontrib-mermaid", + "sphinxcontrib-spelling", ] dev = [ "fastcs-secop[doc]", @@ -75,7 +76,7 @@ omit = [ ] [tool.coverage.report] -fail_under = 100 +# fail_under = 100 # TODO exclude_lines = [ "pragma: no cover", "if TYPE_CHECKING:", diff --git a/ruff.toml b/ruff.toml index 4f22e05..3aa4957 100644 --- a/ruff.toml +++ b/ruff.toml @@ -32,7 +32,7 @@ ignore = [ "D407", # Missing dashed underline after section ("{name}") "D213", # Incompatible with D212 "D203", # Incompatible with D211 - "PLR6301" # Too noisy + "PLR6301", # Too noisy ] [lint.per-file-ignores] "tests/*" = [ diff --git a/src/fastcs_secop/__init__.py b/src/fastcs_secop/__init__.py index 0f00aff..502db22 100644 --- a/src/fastcs_secop/__init__.py +++ b/src/fastcs_secop/__init__.py @@ -9,12 +9,12 @@ from fastcs.transports import EpicsIOCOptions, EpicsPVATransport from fastcs.transports.epics.ca import EpicsCATransport -from fastcs_secop._controllers import SecopController, SecopModuleController from fastcs_secop._util import SecopError +from fastcs_secop.controllers import SecopController logger = getLogger(__name__) -__all__ = ["SecopController", "SecopError", "SecopModuleController"] +__all__ = ["SecopError"] if __name__ == "__main__": # pragma: no cover diff --git a/src/fastcs_secop/_io.py b/src/fastcs_secop/_io.py deleted file mode 100644 index eda0cc3..0000000 --- a/src/fastcs_secop/_io.py +++ /dev/null @@ -1,70 +0,0 @@ -import json -import typing -from dataclasses import dataclass -from logging import getLogger - -from fastcs.attributes import AttributeIO, AttributeIORef, AttrR, AttrW -from fastcs.connections import IPConnection - -from fastcs_secop._util import SecopError - -logger = getLogger(__name__) - -NumberT = typing.TypeVar("NumberT", int, float) - - -@dataclass -class SecopAttributeIORef(AttributeIORef): - module_name: str = "" - accessible_name: str = "" - decode: typing.Callable[[typing.Any], typing.Any] = lambda x: x - encode: typing.Callable[[typing.Any], typing.Any] = lambda x: x - - -class SecopAttributeIO(AttributeIO[NumberT, SecopAttributeIORef]): - def __init__(self, *, connection: IPConnection) -> None: - super().__init__() - - self._connection = connection - - async def update(self, attr: AttrR[NumberT, SecopAttributeIORef]) -> None: - """Read value from device and update the value in FastCS.""" - try: - query = f"read {attr.io_ref.module_name}:{attr.io_ref.accessible_name}\n" - response = await self._connection.send_query(query) - response = response.strip() - - prefix = f"reply {attr.io_ref.module_name}:{attr.io_ref.accessible_name} " - if not response.startswith(prefix): - raise SecopError( - f"Invalid response to 'read' command by SECoP device: '{response}'" - ) - - value = json.loads(response[len(prefix) :])[0] - value = attr.io_ref.decode(value) - await attr.update(value) - except ConnectionError: - # Reconnect will be attempted in a periodic scan task - pass - except Exception: - logger.exception("Exception during update()") - - async def send(self, attr: AttrW[NumberT, SecopAttributeIORef], value: NumberT) -> None: - """Send a value from FastCS to the device.""" - try: - query = f"change {attr.io_ref.module_name}:{attr.io_ref.accessible_name} {value}\n" - - response = await self._connection.send_query(query) - response = response.strip() - - prefix = f"changed {attr.io_ref.module_name}:{attr.io_ref.accessible_name} " - - if not response.startswith(prefix): - raise SecopError( - f"Invalid response to 'change' command by SECoP device: '{response}'" - ) - except ConnectionError: - # Reconnect will be attempted in a periodic scan task - pass - except Exception: - logger.exception("Exception during update()") diff --git a/src/fastcs_secop/_util.py b/src/fastcs_secop/_util.py index e3b9ac8..48981b3 100644 --- a/src/fastcs_secop/_util.py +++ b/src/fastcs_secop/_util.py @@ -1,2 +1,45 @@ +from typing import Any + +import numpy as np +import numpy.typing as npt + + class SecopError(Exception): """Error raised to identify a SECoP protocol or configuration problem.""" + + +def format_string_to_prec(fmt_str: str | None) -> int | None: + """Convert a SECoP format-string specifier to a precision.""" + if fmt_str is None: + return None + + if fmt_str.startswith("%.") and fmt_str.endswith("f"): + return int(fmt_str[2:-1]) + + return None + + +def secop_dtype_to_numpy_dtype(secop_dtype: str) -> npt.DTypeLike: + if secop_dtype == "double": + return np.float64 + elif secop_dtype == "int": + return np.int32 + elif secop_dtype == "bool": + return np.uint8 + else: + raise SecopError(f"Cannot handle SECoP dtype '{secop_dtype}' within array/struct/tuple") + + +def tuple_structured_dtype(datainfo: dict[str, Any]) -> list[tuple[str, npt.DTypeLike]]: + secop_dtypes = [t["type"] for t in datainfo["members"]] + np_dtypes = [secop_dtype_to_numpy_dtype(t) for t in secop_dtypes] + names = [f"e{n}" for n in range(len(datainfo["members"]))] + structured_np_dtype = list(zip(names, np_dtypes, strict=True)) + return structured_np_dtype + + +def struct_structured_dtype(datainfo: dict[str, Any]) -> list[tuple[str, npt.DTypeLike]]: + structured_np_dtype = [ + (k, secop_dtype_to_numpy_dtype(v["type"])) for k, v in datainfo["members"].items() + ] + return structured_np_dtype diff --git a/src/fastcs_secop/_controllers.py b/src/fastcs_secop/controllers.py similarity index 70% rename from src/fastcs_secop/_controllers.py rename to src/fastcs_secop/controllers.py index a93a799..fb21a49 100644 --- a/src/fastcs_secop/_controllers.py +++ b/src/fastcs_secop/controllers.py @@ -1,4 +1,5 @@ -import base64 +"""FastCS controllers for SECoP nodes.""" + import enum import json import typing @@ -6,42 +7,26 @@ from logging import getLogger import numpy as np -import numpy.typing as npt from fastcs.attributes import AttrR, AttrRW from fastcs.connections import IPConnection, IPConnectionSettings from fastcs.controllers import Controller from fastcs.datatypes import Bool, Enum, Float, Int, String, Table, Waveform from fastcs.methods import scan -from fastcs_secop._io import SecopAttributeIO, SecopAttributeIORef -from fastcs_secop._util import SecopError +from fastcs_secop._util import SecopError, format_string_to_prec, struct_structured_dtype +from fastcs_secop.io import ( + SecopAttributeIO, + SecopAttributeIORef, + secop_dtype_to_numpy_dtype, + tuple_structured_dtype, +) logger = getLogger(__name__) -def format_string_to_prec(fmt_str: str | None) -> int | None: - """Convert a SECoP format-string specifier to a precision.""" - if fmt_str is None: - return None - - if fmt_str.startswith("%.") and fmt_str.endswith("f"): - return int(fmt_str[2:-1]) - - return None - - -def secop_dtype_to_numpy_dtype(secop_dtype: str) -> npt.DTypeLike: - if secop_dtype == "double": - return np.float64 - elif secop_dtype == "int": - return np.int32 - elif secop_dtype == "bool": - return np.uint8 - else: - raise SecopError(f"Cannot handle SECoP dtype '{secop_dtype}' within array/struct/tuple") - - class SecopModuleController(Controller): + """FastCS controller for a SECoP module.""" + def __init__( self, *, @@ -66,7 +51,8 @@ def __init__( self._module = module super().__init__(ios=[SecopAttributeIO(connection=connection)]) - async def initialise(self) -> None: + async def initialise(self) -> None: # noqa PLR0912 TODO + """Create attributes for all accessibles in this SECoP module.""" for parameter_name, parameter in self._module["accessibles"].items(): datainfo = parameter["datainfo"] secop_dtype = datainfo["type"] @@ -77,6 +63,13 @@ async def initialise(self) -> None: attr_cls = AttrR if parameter.get("readonly", False) else AttrRW + io_ref = SecopAttributeIORef( + module_name=self._module_name, + accessible_name=parameter_name, + update_period=1.0, + datainfo=datainfo, + ) + if secop_dtype == "command": # TODO: handle commands pass @@ -95,13 +88,7 @@ async def initialise(self) -> None: max_alarm=max_val, prec=format_string_to_prec(datainfo.get("fmtstr", None)) or 6, ), - io_ref=SecopAttributeIORef( - module_name=self._module_name, - accessible_name=parameter_name, - update_period=1.0, - decode=lambda x, s=scale: x * s if s is not None else x, - encode=lambda x, s=scale: round(x / s) if s is not None else x, - ), + io_ref=io_ref, description=parameter.get("description", ""), ), ) @@ -114,11 +101,7 @@ async def initialise(self) -> None: min_alarm=min_val, max_alarm=max_val, ), - io_ref=SecopAttributeIORef( - module_name=self._module_name, - accessible_name=parameter_name, - update_period=1.0, - ), + io_ref=io_ref, description=parameter.get("description", ""), ), ) @@ -127,11 +110,7 @@ async def initialise(self) -> None: parameter_name, attr_cls( Bool(), - io_ref=SecopAttributeIORef( - module_name=self._module_name, - accessible_name=parameter_name, - update_period=1.0, - ), + io_ref=io_ref, description=parameter.get("description", ""), ), ) @@ -143,11 +122,7 @@ async def initialise(self) -> None: parameter_name, attr_cls( Enum(enum_type), - io_ref=SecopAttributeIORef( - module_name=self._module_name, - accessible_name=parameter_name, - update_period=1.0, - ), + io_ref=io_ref, description=parameter.get("description", ""), ), ) @@ -156,11 +131,7 @@ async def initialise(self) -> None: parameter_name, attr_cls( String(), - io_ref=SecopAttributeIORef( - module_name=self._module_name, - accessible_name=parameter_name, - update_period=1.0, - ), + io_ref=io_ref, description=parameter.get("description", ""), ), ) @@ -169,13 +140,7 @@ async def initialise(self) -> None: parameter_name, attr_cls( Waveform(np.uint8, shape=(datainfo["maxbytes"],)), - io_ref=SecopAttributeIORef( - module_name=self._module_name, - accessible_name=parameter_name, - update_period=1.0, - decode=lambda x: np.frombuffer(base64.b64decode(x), dtype=np.uint8), - encode=base64.b64encode, - ), + io_ref=io_ref, description=parameter.get("description", ""), ), ) @@ -183,52 +148,43 @@ async def initialise(self) -> None: inner_dtype = datainfo["members"]["type"] np_inner_dtype = secop_dtype_to_numpy_dtype(inner_dtype) - def decode( - x: list[int | float | bool], t: npt.DTypeLike = np_inner_dtype - ) -> npt.NDArray[np.int32 | np.float64 | np.bool_]: - return np.array(x, dtype=t) - self.add_attribute( parameter_name, attr_cls( Waveform(np_inner_dtype, shape=(datainfo["maxlen"],)), - io_ref=SecopAttributeIORef( - module_name=self._module_name, - accessible_name=parameter_name, - update_period=1.0, - decode=decode, - ), + io_ref=io_ref, description=parameter.get("description", ""), ), ) elif secop_dtype == "tuple": - secop_dtypes = [t["type"] for t in datainfo["members"]] - np_dtypes = [secop_dtype_to_numpy_dtype(t) for t in secop_dtypes] - names = [f"e{n}" for n in range(len(datainfo["members"]))] + structured_dtype = tuple_structured_dtype(datainfo) - structured_dtype = list(zip(names, np_dtypes, strict=True)) - - def decode( - val: list[int | float | bool], t: npt.DTypeLike = structured_dtype - ) -> npt.NDArray[structured_dtype]: - return np.array([tuple(val)], dtype=t) + self.add_attribute( + parameter_name, + attr_cls( + Table(structured_dtype), + io_ref=io_ref, + description=parameter.get("description", ""), + ), + ) + elif secop_dtype == "struct": + structured_dtype = struct_structured_dtype(datainfo) self.add_attribute( parameter_name, attr_cls( Table(structured_dtype), - io_ref=SecopAttributeIORef( - module_name=self._module_name, - accessible_name=parameter_name, - update_period=1.0, - decode=decode, - ), + io_ref=io_ref, description=parameter.get("description", ""), ), ) + else: + raise SecopError(f"Unsupported secop data type '{secop_dtype}") class SecopController(Controller): + """FastCS Controller for a SECoP node.""" + def __init__(self, settings: IPConnectionSettings) -> None: """FastCS Controller for a SECoP node. @@ -243,6 +199,7 @@ def __init__(self, settings: IPConnectionSettings) -> None: super().__init__() async def connect(self) -> None: + """Connect to the SECoP node.""" await self._connection.connect(self._ip_settings) async def deactivate(self) -> None: diff --git a/src/fastcs_secop/io.py b/src/fastcs_secop/io.py new file mode 100644 index 0000000..8c1298f --- /dev/null +++ b/src/fastcs_secop/io.py @@ -0,0 +1,186 @@ +"""Implementation of IO for SECoP accessibles.""" + +import base64 +import json +from dataclasses import dataclass, field +from enum import Enum +from logging import getLogger +from typing import Any, TypeAlias + +import numpy as np +import numpy.typing as npt +from fastcs.attributes import AttributeIO, AttributeIORef, AttrR, AttrW +from fastcs.connections import IPConnection + +from fastcs_secop._util import ( + SecopError, + secop_dtype_to_numpy_dtype, + struct_structured_dtype, + tuple_structured_dtype, +) + +logger = getLogger(__name__) + + +T: TypeAlias = int | float | str | bool | Enum | npt.NDArray[Any] +"""Generic type parameter for SECoP IO.""" + + +async def secop_read( + connection: IPConnection, module_name: str, accessible_name: str +) -> tuple[Any, dict[str, Any]]: + """Read a SECoP accessible. + + Args: + connection: Connection reference, + module_name: Module name + accessible_name: Accessible name + + Returns: + The result of reading from the accessible, after calling :py:obj:`json.loads`. + + Raises: + SecopError: If a valid response was not received + + """ + query = f"read {module_name}:{accessible_name}\n" + response = await connection.send_query(query) + response = response.strip() + + prefix = f"reply {module_name}:{accessible_name} " + if not response.startswith(prefix): + raise SecopError(f"Invalid response to 'read' command by SECoP device: '{response}'") + + value, metadata = json.loads(response[len(prefix) :]) + return value, metadata + + +async def secop_change( + connection: IPConnection, module_name: str, accessible_name: str, encoded_value: str +) -> None: + """Change a SECoP accessible. + + Args: + connection: Connection reference, + module_name: Module name + accessible_name: Accessible name + encoded_value: Value to set (as a raw string ready for transport). + + Raises: + SecopError: If a valid response was not received + + """ + query = f"change {module_name}:{accessible_name} {encoded_value}\n" + + response = await connection.send_query(query) + response = response.strip() + + prefix = f"changed {module_name}:{accessible_name} " + + if not response.startswith(prefix): + raise SecopError(f"Invalid response to 'change' command by SECoP device: '{response}'") + + +@dataclass +class SecopAttributeIORef(AttributeIORef): + """AttributeIO parameters for a SECoP parameter (accessible).""" + + module_name: str = "" + accessible_name: str = "" + datainfo: dict[str, Any] = field(default_factory=dict) + + +class SecopAttributeIO(AttributeIO[T, SecopAttributeIORef]): + """IO for a SECoP parameter of any type other than 'command'.""" + + def __init__(self, *, connection: IPConnection) -> None: + """IO for a SECoP parameter of any type other than 'command'.""" + super().__init__() + + self._connection = connection + + def decode(self, value: Any, datainfo: dict[str, Any]) -> T: # noqa ANN401 + """Decode the transported value into a python datatype. + + Args: + value: The value to decode. This is the result of calling :py:obj:`json.loads` + on the raw transported bytes. + datainfo: The SECoP ``datainfo`` dictionary for this attribute. + + Returns: + Python datatype representation of the transported value. + + """ + match datainfo["type"]: + case "int" | "bool" | "double" | "string" | "enum": + return value + case "scaled": + return value * datainfo["scale"] + case "blob": + return np.frombuffer(base64.b64decode(value), dtype=np.uint8) + case "array": + inner_np_dtype = secop_dtype_to_numpy_dtype(datainfo["members"]["type"]) + return np.array(value, dtype=inner_np_dtype) + case "tuple": + structured_np_dtype = tuple_structured_dtype(datainfo) + return np.array([tuple(value)], dtype=structured_np_dtype) + case "struct": + structured_np_dtype = struct_structured_dtype(datainfo) + arr = np.zeros(shape=(1,), dtype=structured_np_dtype) + for k, v in value.items(): + arr[0][k] = v + return arr + case _: + raise SecopError(f"Cannot handle SECoP dtype '{datainfo['type']}'") + + def encode(self, value: T, datainfo: dict[str, Any]) -> str: + """Encode the transported value to a string for transport. + + Args: + value: The value to encode. + datainfo: The SECoP ``datainfo`` dictionary for this attribute. + + """ + match datainfo["type"]: + case "int" | "bool" | "double" | "string" | "enum": + return json.dumps(value) + case "scaled": + return json.dumps(round(value / datainfo["scale"])) + case "blob": + assert isinstance(value, np.ndarray) + return json.dumps(base64.b64encode("".join(chr(c) for c in value).encode("utf-8"))) + case "array" | "tuple": + assert isinstance(value, np.ndarray) + return json.dumps(value.tolist()) + case _: + raise SecopError(f"Cannot handle SECoP dtype '{datainfo['type']}'") + + async def update(self, attr: AttrR[T, SecopAttributeIORef]) -> None: + """Read value from device and update the value in FastCS.""" + try: + raw_value, _ = await secop_read( + self._connection, attr.io_ref.module_name, attr.io_ref.accessible_name + ) + value = self.decode(raw_value, attr.io_ref.datainfo) + await attr.update(value) + except ConnectionError: + # Reconnect will be attempted in a periodic scan task + pass + except Exception: + logger.exception("Exception during update()") + + async def send(self, attr: AttrW[T, SecopAttributeIORef], value: T) -> None: + """Send a value from FastCS to the device.""" + try: + encoded_value = self.encode(value, attr.io_ref.datainfo) + await secop_change( + self._connection, + attr.io_ref.module_name, + attr.io_ref.accessible_name, + encoded_value, + ) + except ConnectionError: + # Reconnect will be attempted in a periodic scan task + pass + except Exception: + logger.exception("Exception during send()") diff --git a/tests/emulators/simple_secop/device.py b/tests/emulators/simple_secop/device.py index 69d35aa..3a293ac 100644 --- a/tests/emulators/simple_secop/device.py +++ b/tests/emulators/simple_secop/device.py @@ -123,6 +123,18 @@ def __init__(self): dtype="tuple", extra_datainfo={"members": [{"type": "int"}, {"type": "double"}, {"type": "bool"}]}, ), + "struct": Parameter( + {"answer": 42, "pi": math.pi, "on_fire": True}, + desc="a struct of int, float, bool", + dtype="struct", + extra_datainfo={ + "members": { + "answer": {"type": "int"}, + "pi": {"type": "double"}, + "on_fire": {"type": "bool"}, + } + }, + ), } self.description = "a module with one accessible of each possible dtype" diff --git a/tests/test_against_emulator.py b/tests/test_against_emulator.py index 05ae2cd..fdd5625 100644 --- a/tests/test_against_emulator.py +++ b/tests/test_against_emulator.py @@ -12,7 +12,7 @@ from fastcs.connections import IPConnectionSettings from fastcs.logging import LogLevel, configure_logging -from fastcs_secop import SecopController +from fastcs_secop.controllers import SecopController configure_logging(level=LogLevel.TRACE) @@ -120,6 +120,13 @@ async def test_attributes_created_for_enum_datatype(self, controller): [(1, 5.678, 1)], dtype=[("e0", np.int32), ("e1", np.float64), ("e2", np.uint8)] ), ), + ( + "struct", + np.array( + [(42, math.pi, 1)], + dtype=[("answer", np.int32), ("pi", np.float64), ("on_fire", np.uint8)], + ), + ), ], ) async def test_attributes_created_for_array_datatype( diff --git a/tests/test_controller.py b/tests/test_controller.py index dcfe979..4a7c3b0 100644 --- a/tests/test_controller.py +++ b/tests/test_controller.py @@ -5,7 +5,7 @@ from fastcs.connections import IPConnectionSettings from fastcs_secop import SecopError -from fastcs_secop._controllers import ( +from fastcs_secop.controllers import ( SecopController, format_string_to_prec, secop_dtype_to_numpy_dtype, From a69597af554a51c3544a9fc310c60c10fe152f1b Mon Sep 17 00:00:00 2001 From: Tom Willemsen Date: Fri, 26 Dec 2025 20:53:54 +0000 Subject: [PATCH 11/19] refactoring --- README.md | 4 + doc/index.md | 1 + doc/limitations.md | 42 +++++++ doc/spelling_wordlist.txt | 7 ++ src/fastcs_secop/__init__.py | 24 +++- src/fastcs_secop/_util.py | 31 +++-- src/fastcs_secop/controllers.py | 165 +++++++++++++++++++------ src/fastcs_secop/io.py | 81 +++++++++--- tests/emulators/simple_secop/device.py | 16 ++- tests/test_against_emulator.py | 22 +++- tests/test_controller.py | 11 +- 11 files changed, 327 insertions(+), 77 deletions(-) create mode 100644 README.md create mode 100644 doc/limitations.md diff --git a/README.md b/README.md new file mode 100644 index 0000000..ac01021 --- /dev/null +++ b/README.md @@ -0,0 +1,4 @@ +# `fastcs-secop` + +Support for [SECoP](https://sampleenvironment.github.io/secop-site/intro/index.html) devices using +[FastCS](https://diamondlightsource.github.io/FastCS/main/index.html). diff --git a/doc/index.md b/doc/index.md index a15fb01..b3cbfec 100644 --- a/doc/index.md +++ b/doc/index.md @@ -7,5 +7,6 @@ using {external+fastcs:doc}`FastCS `. :titlesonly: :caption: Reference +limitations _api ``` diff --git a/doc/limitations.md b/doc/limitations.md new file mode 100644 index 0000000..0b54834 --- /dev/null +++ b/doc/limitations.md @@ -0,0 +1,42 @@ +# Limitations + +There are some elements of the {external+secop:doc}`SECoP specification ` that +{py:obj}`fastcs_secop` does not currently support. These are detailed below. + +## Data-type limitations + +### Enums within arrays/structs/tuples + +An enum-type element *within* an array/struct/tuple is treated as its corresponding integer value and loses name-based +functionality. + +Rationale: FastCS does not provide a way to describe an enum nested within a {py:obj}`~fastcs.datatypes.table.Table` +or {py:obj}`~fastcs.datatypes.waveform.Waveform`. + +### Nested arrays/structs/tuples + +Arrays/structs/tuples nested inside another array/struct/tuple are not supported. Arrays, structs and tuples can only +be made from 'simple' data types (double, int, bool, enum, string). + +Rationale: most FastCS transports cannot support these nested datatypes easily. Nested arrays create the possibility +of ragged arrays which cannot be expressed using standard {py:obj}`numpy` datatypes. + +Workaround: Use {py:obj}`fastcs_secop.SecopQuirks` to either skip the accessible, or read it in +'raw' mode which treats the SECoP JSON response as a string to be interpreted downstream, rather than deserialising it. + +## Transport-specific limitations + +FastCS supports multiple transport types (e.g. EPICS CA, EPICS PVA, Tango, REST, ...). However, not all datatypes are +supported using all transports. Notably, EPICS CA lacks support for the {py:obj}`~fastcs.datatypes.table.Table` type, +which is used to implement structs and tuples. + +Workaround: Use {py:obj}`fastcs_secop.SecopQuirks` to either skip the accessible, or read it in +'raw' mode, which treats the SECoP JSON response as a string to be interpreted downstream, rather than deserialising it. +A {py:obj}`~collections.defaultdict` can be used to specify reading *all* arrays, structs or tuples in raw mode. + +## Asynchronous updates + +Asynchronous updates are not supported by {py:obj}`fastcs_secop`. They are turned off using a +{external+secop:doc}`deactivate message ` at connection time. + +Rationale: FastCS does not currently provide infrastructure to handle asynchronous messages. diff --git a/doc/spelling_wordlist.txt b/doc/spelling_wordlist.txt index 81d8ebe..cef5c9b 100644 --- a/doc/spelling_wordlist.txt +++ b/doc/spelling_wordlist.txt @@ -1,6 +1,13 @@ accessible accessibles +datatype +datatypes deserialised +deserialising +enum +enums SECoP +struct +structs subcontroller subcontrollers diff --git a/src/fastcs_secop/__init__.py b/src/fastcs_secop/__init__.py index 502db22..b8a2316 100644 --- a/src/fastcs_secop/__init__.py +++ b/src/fastcs_secop/__init__.py @@ -1,6 +1,8 @@ """SECoP support using FastCS.""" +import asyncio import logging +from collections import defaultdict from logging import getLogger from fastcs.connections import IPConnectionSettings @@ -9,12 +11,12 @@ from fastcs.transports import EpicsIOCOptions, EpicsPVATransport from fastcs.transports.epics.ca import EpicsCATransport -from fastcs_secop._util import SecopError +from fastcs_secop._util import SecopError, SecopQuirks from fastcs_secop.controllers import SecopController logger = getLogger(__name__) -__all__ = ["SecopError"] +__all__ = ["SecopError", "SecopQuirks"] if __name__ == "__main__": # pragma: no cover @@ -22,12 +24,26 @@ logging.basicConfig(level=LogLevel.DEBUG) + asyncio.get_event_loop().slow_callback_duration = 1000 + epics_options = EpicsIOCOptions(pv_prefix="TE:NDW2922:SECOP") epics_ca = EpicsCATransport(epicsca=epics_options) epics_pva = EpicsPVATransport(epicspva=epics_options) + quirks = defaultdict( + lambda: SecopQuirks(raw_tuple=False, raw_struct=False, max_description_length=40) + ) + quirks["valve_controller._domains_to_extract"] = SecopQuirks(raw_array=True) + quirks["valve_controller._terminal_values"] = SecopQuirks(raw_struct=True) + + LEWIS = 57677 + DOCKER_GASFLOW = 10801 + fastcs = FastCS( - SecopController(settings=IPConnectionSettings(ip="127.0.0.1", port=57677)), + SecopController( + settings=IPConnectionSettings(ip="127.0.0.1", port=LEWIS), + quirks=quirks, + ), [epics_pva], ) - fastcs.run(interactive=False) + fastcs.run(interactive=True) diff --git a/src/fastcs_secop/_util.py b/src/fastcs_secop/_util.py index 48981b3..c08d560 100644 --- a/src/fastcs_secop/_util.py +++ b/src/fastcs_secop/_util.py @@ -1,9 +1,19 @@ +from dataclasses import dataclass from typing import Any import numpy as np import numpy.typing as npt +@dataclass(frozen=True) +class SecopQuirks: + skip: bool = False + raw_array: bool = False + raw_tuple: bool = False + raw_struct: bool = False + max_description_length: int | None = None + + class SecopError(Exception): """Error raised to identify a SECoP protocol or configuration problem.""" @@ -19,19 +29,26 @@ def format_string_to_prec(fmt_str: str | None) -> int | None: return None -def secop_dtype_to_numpy_dtype(secop_dtype: str) -> npt.DTypeLike: - if secop_dtype == "double": +def secop_dtype_to_numpy_dtype(secop_datainfo: dict[str, Any]) -> npt.DTypeLike: + dtype = secop_datainfo["type"] + if dtype == "double": return np.float64 - elif secop_dtype == "int": + elif dtype == "int": return np.int32 - elif secop_dtype == "bool": + elif dtype == "bool": return np.uint8 + elif dtype == "enum": + return np.int32 + elif dtype == "string": + return f" list[tuple[str, npt.DTypeLike]]: - secop_dtypes = [t["type"] for t in datainfo["members"]] + secop_dtypes = [t for t in datainfo["members"]] np_dtypes = [secop_dtype_to_numpy_dtype(t) for t in secop_dtypes] names = [f"e{n}" for n in range(len(datainfo["members"]))] structured_np_dtype = list(zip(names, np_dtypes, strict=True)) @@ -40,6 +57,6 @@ def tuple_structured_dtype(datainfo: dict[str, Any]) -> list[tuple[str, npt.DTyp def struct_structured_dtype(datainfo: dict[str, Any]) -> list[tuple[str, npt.DTypeLike]]: structured_np_dtype = [ - (k, secop_dtype_to_numpy_dtype(v["type"])) for k, v in datainfo["members"].items() + (k, secop_dtype_to_numpy_dtype(v)) for k, v in datainfo["members"].items() ] return structured_np_dtype diff --git a/src/fastcs_secop/controllers.py b/src/fastcs_secop/controllers.py index fb21a49..40ea20a 100644 --- a/src/fastcs_secop/controllers.py +++ b/src/fastcs_secop/controllers.py @@ -13,10 +13,13 @@ from fastcs.datatypes import Bool, Enum, Float, Int, String, Table, Waveform from fastcs.methods import scan +from fastcs_secop import SecopQuirks from fastcs_secop._util import SecopError, format_string_to_prec, struct_structured_dtype from fastcs_secop.io import ( SecopAttributeIO, SecopAttributeIORef, + SecopRawAttributeIO, + SecopRawAttributeIORef, secop_dtype_to_numpy_dtype, tuple_structured_dtype, ) @@ -33,6 +36,7 @@ def __init__( connection: IPConnection, module_name: str, module: dict[str, typing.Any], + quirks: typing.Mapping[str, SecopQuirks], ) -> None: """FastCS controller for a SECoP module. @@ -45,15 +49,29 @@ def __init__( module: A deserialised description, in the :external+secop:doc:`SECoP over-the-wire format `, of this module. + quirks: dict-like object of :py:obj:`~fastcs_secop.SecopQuirks` that affects how + attributes are processed. """ self._module_name = module_name self._module = module - super().__init__(ios=[SecopAttributeIO(connection=connection)]) + self._quirks = quirks + super().__init__( + ios=[ + SecopAttributeIO(connection=connection), + SecopRawAttributeIO(connection=connection), + ] + ) async def initialise(self) -> None: # noqa PLR0912 TODO """Create attributes for all accessibles in this SECoP module.""" for parameter_name, parameter in self._module["accessibles"].items(): + quirks = self._quirks.get(f"{self._module_name}.{parameter_name}", SecopQuirks()) + + if quirks.skip: + continue + + logger.debug("Creating attribute for parameter %s", parameter_name) datainfo = parameter["datainfo"] secop_dtype = datainfo["type"] @@ -61,6 +79,8 @@ async def initialise(self) -> None: # noqa PLR0912 TODO max_val = datainfo.get("max") scale = datainfo.get("scale") + description = parameter.get("description", "")[: quirks.max_description_length] + attr_cls = AttrR if parameter.get("readonly", False) else AttrRW io_ref = SecopAttributeIORef( @@ -89,7 +109,7 @@ async def initialise(self) -> None: # noqa PLR0912 TODO prec=format_string_to_prec(datainfo.get("fmtstr", None)) or 6, ), io_ref=io_ref, - description=parameter.get("description", ""), + description=description, ), ) elif secop_dtype == "int": @@ -102,7 +122,7 @@ async def initialise(self) -> None: # noqa PLR0912 TODO max_alarm=max_val, ), io_ref=io_ref, - description=parameter.get("description", ""), + description=description, ), ) elif secop_dtype == "bool": @@ -111,11 +131,10 @@ async def initialise(self) -> None: # noqa PLR0912 TODO attr_cls( Bool(), io_ref=io_ref, - description=parameter.get("description", ""), + description=description, ), ) elif secop_dtype == "enum": - # TODO: Bug - this doesn't work properly with PVA? enum_type = enum.Enum("enum_type", datainfo["members"]) self.add_attribute( @@ -123,7 +142,7 @@ async def initialise(self) -> None: # noqa PLR0912 TODO attr_cls( Enum(enum_type), io_ref=io_ref, - description=parameter.get("description", ""), + description=description, ), ) elif secop_dtype == "string": @@ -132,7 +151,7 @@ async def initialise(self) -> None: # noqa PLR0912 TODO attr_cls( String(), io_ref=io_ref, - description=parameter.get("description", ""), + description=description, ), ) elif secop_dtype == "blob": @@ -141,43 +160,91 @@ async def initialise(self) -> None: # noqa PLR0912 TODO attr_cls( Waveform(np.uint8, shape=(datainfo["maxbytes"],)), io_ref=io_ref, - description=parameter.get("description", ""), + description=description, ), ) elif secop_dtype == "array": - inner_dtype = datainfo["members"]["type"] - np_inner_dtype = secop_dtype_to_numpy_dtype(inner_dtype) - - self.add_attribute( - parameter_name, - attr_cls( - Waveform(np_inner_dtype, shape=(datainfo["maxlen"],)), - io_ref=io_ref, - description=parameter.get("description", ""), - ), - ) + if self._quirks.get( + f"{self._module_name}.{parameter_name}", SecopQuirks() + ).raw_array: + self.add_attribute( + parameter_name, + attr_cls( + String(65536), + io_ref=SecopRawAttributeIORef( + module_name=self._module_name, + accessible_name=parameter_name, + update_period=1.0, + ), + description=description, + ), + ) + else: + inner_dtype = datainfo["members"] + np_inner_dtype = secop_dtype_to_numpy_dtype(inner_dtype) + + self.add_attribute( + parameter_name, + attr_cls( + Waveform(np_inner_dtype, shape=(datainfo["maxlen"],)), + io_ref=io_ref, + description=description, + ), + ) elif secop_dtype == "tuple": - structured_dtype = tuple_structured_dtype(datainfo) - - self.add_attribute( - parameter_name, - attr_cls( - Table(structured_dtype), - io_ref=io_ref, - description=parameter.get("description", ""), - ), - ) + if self._quirks.get( + f"{self._module_name}.{parameter_name}", SecopQuirks() + ).raw_tuple: + self.add_attribute( + parameter_name, + attr_cls( + String(65536), + io_ref=SecopRawAttributeIORef( + module_name=self._module_name, + accessible_name=parameter_name, + update_period=1.0, + ), + description=description, + ), + ) + else: + structured_dtype = tuple_structured_dtype(datainfo) + + self.add_attribute( + parameter_name, + attr_cls( + Table(structured_dtype), + io_ref=io_ref, + description=description, + ), + ) elif secop_dtype == "struct": - structured_dtype = struct_structured_dtype(datainfo) - - self.add_attribute( - parameter_name, - attr_cls( - Table(structured_dtype), - io_ref=io_ref, - description=parameter.get("description", ""), - ), - ) + if self._quirks.get( + f"{self._module_name}.{parameter_name}", SecopQuirks() + ).raw_struct: + self.add_attribute( + parameter_name, + attr_cls( + String(65536), + io_ref=SecopRawAttributeIORef( + module_name=self._module_name, + accessible_name=parameter_name, + update_period=1.0, + ), + description=description, + ), + ) + else: + structured_dtype = struct_structured_dtype(datainfo) + + self.add_attribute( + parameter_name, + attr_cls( + Table(structured_dtype), + io_ref=io_ref, + description=description, + ), + ) else: raise SecopError(f"Unsupported secop data type '{secop_dtype}") @@ -185,16 +252,27 @@ async def initialise(self) -> None: # noqa PLR0912 TODO class SecopController(Controller): """FastCS Controller for a SECoP node.""" - def __init__(self, settings: IPConnectionSettings) -> None: + def __init__( + self, settings: IPConnectionSettings, quirks: typing.Mapping[str, SecopQuirks] | None = None + ) -> None: """FastCS Controller for a SECoP node. Args: settings: The communication settings (e.g. IP address, port) at which the SECoP node is reachable. + quirks: :py:obj:`dict`-like object of :py:obj:`~fastcs_secop.SecopQuirks` + that affects how attributes are processed. + + Quirks may be applied to a module, keyed by ``module_name``, or to an + individual parameter, keyed by ``module_name.parameter_name``. + + Hint: use a :py:obj:`~collections.defaultdict` to + specify quirks that apply to all attributes and modules. """ self._ip_settings = settings self._connection = IPConnection() + self.quirks = quirks if quirks is not None else {} super().__init__() @@ -292,15 +370,20 @@ async def initialise(self) -> None: description = descriptor["description"] equipment_id = descriptor["equipment_id"] - print(f"SECoP equipment_id = '{equipment_id}', description = '{description}'") + logger.info("SECoP equipment_id = '%s', description = '%s'", equipment_id, description) + logger.debug("descriptor = %s", json.dumps(descriptor, indent=2)) modules = descriptor["modules"] for module_name, module in modules.items(): + if self.quirks.get(module_name, SecopQuirks()).skip: + continue + logger.debug("Creating subcontroller for module %s", module_name) module_controller = SecopModuleController( connection=self._connection, module_name=module_name, module=module, + quirks=self.quirks, ) await module_controller.initialise() self.add_sub_controller(name=module_name, sub_controller=module_controller) diff --git a/src/fastcs_secop/io.py b/src/fastcs_secop/io.py index 8c1298f..d67d725 100644 --- a/src/fastcs_secop/io.py +++ b/src/fastcs_secop/io.py @@ -5,7 +5,7 @@ from dataclasses import dataclass, field from enum import Enum from logging import getLogger -from typing import Any, TypeAlias +from typing import Any, TypeAlias, cast import numpy as np import numpy.typing as npt @@ -26,9 +26,7 @@ """Generic type parameter for SECoP IO.""" -async def secop_read( - connection: IPConnection, module_name: str, accessible_name: str -) -> tuple[Any, dict[str, Any]]: +async def secop_read(connection: IPConnection, module_name: str, accessible_name: str) -> str: """Read a SECoP accessible. Args: @@ -51,8 +49,7 @@ async def secop_read( if not response.startswith(prefix): raise SecopError(f"Invalid response to 'read' command by SECoP device: '{response}'") - value, metadata = json.loads(response[len(prefix) :]) - return value, metadata + return response[len(prefix) :] async def secop_change( @@ -90,6 +87,14 @@ class SecopAttributeIORef(AttributeIORef): datainfo: dict[str, Any] = field(default_factory=dict) +@dataclass +class SecopRawAttributeIORef(AttributeIORef): + """RawAttributeIO parameters for a SECoP parameter (accessible).""" + + module_name: str = "" + accessible_name: str = "" + + class SecopAttributeIO(AttributeIO[T, SecopAttributeIORef]): """IO for a SECoP parameter of any type other than 'command'.""" @@ -99,27 +104,29 @@ def __init__(self, *, connection: IPConnection) -> None: self._connection = connection - def decode(self, value: Any, datainfo: dict[str, Any]) -> T: # noqa ANN401 + def decode(self, value: str, datainfo: dict[str, Any], attr: AttrR[T]) -> T: # noqa ANN401 """Decode the transported value into a python datatype. Args: - value: The value to decode. This is the result of calling :py:obj:`json.loads` - on the raw transported bytes. + value: The value to decode (the raw transported string) datainfo: The SECoP ``datainfo`` dictionary for this attribute. Returns: Python datatype representation of the transported value. """ + value, *_ = json.loads(value) match datainfo["type"]: - case "int" | "bool" | "double" | "string" | "enum": + case "int" | "bool" | "double" | "string": return value + case "enum": + return attr.dtype(cast(int, value)) case "scaled": return value * datainfo["scale"] case "blob": return np.frombuffer(base64.b64decode(value), dtype=np.uint8) case "array": - inner_np_dtype = secop_dtype_to_numpy_dtype(datainfo["members"]["type"]) + inner_np_dtype = secop_dtype_to_numpy_dtype(datainfo["members"]) return np.array(value, dtype=inner_np_dtype) case "tuple": structured_np_dtype = tuple_structured_dtype(datainfo) @@ -127,7 +134,7 @@ def decode(self, value: Any, datainfo: dict[str, Any]) -> T: # noqa ANN401 case "struct": structured_np_dtype = struct_structured_dtype(datainfo) arr = np.zeros(shape=(1,), dtype=structured_np_dtype) - for k, v in value.items(): + for k, v in cast(dict[str, Any], value).items(): arr[0][k] = v return arr case _: @@ -158,10 +165,10 @@ def encode(self, value: T, datainfo: dict[str, Any]) -> str: async def update(self, attr: AttrR[T, SecopAttributeIORef]) -> None: """Read value from device and update the value in FastCS.""" try: - raw_value, _ = await secop_read( + raw_value = await secop_read( self._connection, attr.io_ref.module_name, attr.io_ref.accessible_name ) - value = self.decode(raw_value, attr.io_ref.datainfo) + value = self.decode(raw_value, attr.io_ref.datainfo, attr) await attr.update(value) except ConnectionError: # Reconnect will be attempted in a periodic scan task @@ -184,3 +191,49 @@ async def send(self, attr: AttrW[T, SecopAttributeIORef], value: T) -> None: pass except Exception: logger.exception("Exception during send()") + + +class SecopRawAttributeIO(AttributeIO[str, SecopRawAttributeIORef]): + """Raw IO for a SECoP parameter of any type other than 'command'. + + For "raw" IO, no serialization/deserialization is performed. All values are transmitted + to/from FastCS as strings. It is up to the client to interpret those strings correctly. + + This is intended as a fallback mode for transports which cannot represent complex + data types. + + """ + + def __init__(self, *, connection: IPConnection) -> None: + """IO for a SECoP parameter of any type other than 'command'.""" + super().__init__() + + self._connection = connection + + async def update(self, attr: AttrR[str, SecopRawAttributeIORef]) -> None: + """Read value from device and update the value in FastCS.""" + try: + raw_value = await secop_read( + self._connection, attr.io_ref.module_name, attr.io_ref.accessible_name + ) + await attr.update(raw_value) + except ConnectionError: + # Reconnect will be attempted in a periodic scan task + pass + except Exception: + logger.exception("Exception during update()") + + async def send(self, attr: AttrW[str, SecopRawAttributeIORef], value: str) -> None: + """Send a value from FastCS to the device.""" + try: + await secop_change( + self._connection, + attr.io_ref.module_name, + attr.io_ref.accessible_name, + value, + ) + except ConnectionError: + # Reconnect will be attempted in a periodic scan task + pass + except Exception: + logger.exception("Exception during send()") diff --git a/tests/emulators/simple_secop/device.py b/tests/emulators/simple_secop/device.py index 3a293ac..0da9078 100644 --- a/tests/emulators/simple_secop/device.py +++ b/tests/emulators/simple_secop/device.py @@ -118,13 +118,21 @@ def __init__(self): extra_datainfo={"maxlen": 512, "members": {"type": "bool"}}, ), "tuple": Parameter( - [1, 5.678, True], + [1, 5.678, True, "hiya", 5], desc="a tuple of int, float, bool", dtype="tuple", - extra_datainfo={"members": [{"type": "int"}, {"type": "double"}, {"type": "bool"}]}, + extra_datainfo={ + "members": [ + {"type": "int"}, + {"type": "double"}, + {"type": "bool"}, + {"type": "string"}, + {"type": "enum"}, + ] + }, ), "struct": Parameter( - {"answer": 42, "pi": math.pi, "on_fire": True}, + {"answer": 42, "pi": math.pi, "on_fire": True, "status": "chillin'", "mode": 1}, desc="a struct of int, float, bool", dtype="struct", extra_datainfo={ @@ -132,6 +140,8 @@ def __init__(self): "answer": {"type": "int"}, "pi": {"type": "double"}, "on_fire": {"type": "bool"}, + "status": {"type": "string"}, + "mode": {"type": "enum"}, } }, ), diff --git a/tests/test_against_emulator.py b/tests/test_against_emulator.py index fdd5625..c9bf429 100644 --- a/tests/test_against_emulator.py +++ b/tests/test_against_emulator.py @@ -44,7 +44,8 @@ async def controller(): settings=IPConnectionSettings( ip="127.0.0.1", port=57677, - ) + ), + quirks={}, ) for _ in range(100): @@ -117,14 +118,27 @@ async def test_attributes_created_for_enum_datatype(self, controller): ( "tuple", np.array( - [(1, 5.678, 1)], dtype=[("e0", np.int32), ("e1", np.float64), ("e2", np.uint8)] + [(1, 5.678, 1, "hiya", 5)], + dtype=[ + ("e0", np.int32), + ("e1", np.float64), + ("e2", np.uint8), + ("e3", " Date: Mon, 29 Dec 2025 11:39:44 +0000 Subject: [PATCH 12/19] refactoring --- .github/workflows/Lint-and-test.yml | 14 +++--- .github/workflows/documentation.yml | 13 +++-- .gitignore | 2 + doc/conf.py | 4 +- doc/index.md | 8 ++++ doc/limitations.md | 39 ++++++++++----- doc/spelling_wordlist.txt | 1 + doc/transports/epics_ca.md | 26 ++++++++++ doc/transports/epics_pva.md | 34 ++++++++++++++ doc/transports/tango.md | 18 +++++++ examples/epics_ca.py | 40 ++++++++++++++++ examples/epics_pva.py | 45 ++++++++++++++++++ pyproject.toml | 12 ++--- src/fastcs_secop/__init__.py | 44 ----------------- src/fastcs_secop/_util.py | 42 ++++++++++++++++- src/fastcs_secop/controllers.py | 73 ++++++++++++----------------- src/fastcs_secop/io.py | 21 +++++++-- 17 files changed, 305 insertions(+), 131 deletions(-) create mode 100644 doc/transports/epics_ca.md create mode 100644 doc/transports/epics_pva.md create mode 100644 doc/transports/tango.md create mode 100644 examples/epics_ca.py create mode 100644 examples/epics_pva.py diff --git a/.github/workflows/Lint-and-test.yml b/.github/workflows/Lint-and-test.yml index c5b83df..f44f9c5 100644 --- a/.github/workflows/Lint-and-test.yml +++ b/.github/workflows/Lint-and-test.yml @@ -10,19 +10,19 @@ jobs: fail-fast: false steps: - uses: actions/checkout@v6 - - uses: actions/setup-python@v6 + - uses: astral-sh/setup-uv@v7 with: python-version: ${{ matrix.version }} - name: install requirements - run: pip install -e .[dev] + run: uv sync --extra dev - name: run ruff check - run: python -m ruff check - - name: run ruff format - run: python -m ruff format --check + run: uv run ruff check + - name: run ruff format --check + run: uv run ruff format --check - name: run pyright - run: python -m pyright + run: uv run pyright - name: run pytest - run: python -m pytest + run: uv run pytest results: if: ${{ always() }} runs-on: ubuntu-latest diff --git a/.github/workflows/documentation.yml b/.github/workflows/documentation.yml index 3dbc893..e3cf686 100644 --- a/.github/workflows/documentation.yml +++ b/.github/workflows/documentation.yml @@ -9,16 +9,15 @@ jobs: - uses: actions/checkout@v6 with: fetch-depth: 0 - - uses: actions/setup-python@v5 + - uses: astral-sh/setup-uv@v7 with: - python-version: '3.12' - - name: Install dependencies - run: | - pip install .[doc] + python-version: "3.12" + - name: install requirements + run: uv sync --extra doc - name: Sphinx build - run: sphinx-build -E -a -W --keep-going doc _build + run: uv run sphinx-build -E -a -W --keep-going doc _build - name: run spellcheck - run: sphinx-build -E -a -W --keep-going -b spelling doc _build + run: uv run sphinx-build -E -a -W --keep-going -b spelling doc _build - name: Upload artifact uses: actions/upload-artifact@v4 with: diff --git a/.gitignore b/.gitignore index d1ecc6f..3b14865 100644 --- a/.gitignore +++ b/.gitignore @@ -164,3 +164,5 @@ doc/generated/* _build/ src/fastcs_secop/version.py + +output.bob diff --git a/doc/conf.py b/doc/conf.py index 00cfc59..c36a77a 100644 --- a/doc/conf.py +++ b/doc/conf.py @@ -9,14 +9,14 @@ import os import sys -import fastcs_secop # noqa: F401 +from fastcs_secop.version import version sys.path.insert(0, os.path.abspath("../src")) project = "fastcs-secop" copyright = "" author = "ISIS Experiment Controls" -release = "0.1" +release = version # -- General configuration --------------------------------------------------- # https://www.sphinx-doc.org/en/master/usage/configuration.html#general-configuration diff --git a/doc/index.md b/doc/index.md index b3cbfec..236d423 100644 --- a/doc/index.md +++ b/doc/index.md @@ -3,6 +3,14 @@ The {py:obj}`fastcs_secop` library implements support for the {external+secop:doc}`SECoP protocol ` using {external+fastcs:doc}`FastCS `. +```{toctree} +:titlesonly: +:caption: Transports +:glob: + +transports/* +``` + ```{toctree} :titlesonly: :caption: Reference diff --git a/doc/limitations.md b/doc/limitations.md index 0b54834..b02eaaa 100644 --- a/doc/limitations.md +++ b/doc/limitations.md @@ -3,37 +3,50 @@ There are some elements of the {external+secop:doc}`SECoP specification ` that {py:obj}`fastcs_secop` does not currently support. These are detailed below. +{#limitations_dtype} ## Data-type limitations +{#limitations_float_format} +### Float formatting + +For double and scaled type parameters, the format string (`fmtstr`) is interpreted only if it is in the `f` format. `g` +and `e` formats are ignored. + +Rationale: FastCS provides a `precision` argument to transports, which represents a number of decimal places. +Other formats are not currently representable in FastCS. + +{#limitations_enum} ### Enums within arrays/structs/tuples An enum-type element *within* an array/struct/tuple is treated as its corresponding integer value and loses name-based functionality. Rationale: FastCS does not provide a way to describe an enum nested within a {py:obj}`~fastcs.datatypes.table.Table` -or {py:obj}`~fastcs.datatypes.waveform.Waveform`. +or {py:obj}`~fastcs.datatypes.waveform.Waveform`. Most transports also cannot describe this. +{#limitations_nested_complex} ### Nested arrays/structs/tuples Arrays/structs/tuples nested inside another array/struct/tuple are not supported. Arrays, structs and tuples can only be made from 'simple' data types (double, int, bool, enum, string). -Rationale: most FastCS transports cannot support these nested datatypes easily. Nested arrays create the possibility -of ragged arrays which cannot be expressed using standard {py:obj}`numpy` datatypes. - -Workaround: Use {py:obj}`fastcs_secop.SecopQuirks` to either skip the accessible, or read it in -'raw' mode which treats the SECoP JSON response as a string to be interpreted downstream, rather than deserialising it. +Nested arrays create the possibility of ragged arrays, which cannot be expressed using standard {py:obj}`numpy` +datatypes and are not representable using FastCS's current data types. -## Transport-specific limitations +In principle, the following types could be supported in future (but are not supported currently): +- Arrays of structs or tuples, using the {py:obj}`~fastcs.datatypes.table.Table` type (for transports that support +the {py:obj}`~fastcs.datatypes.table.Table` FastCS type). +- Nested combinations of structs and tuples, by flattening (for transports that support +the {py:obj}`~fastcs.datatypes.table.Table` FastCS type). -FastCS supports multiple transport types (e.g. EPICS CA, EPICS PVA, Tango, REST, ...). However, not all datatypes are -supported using all transports. Notably, EPICS CA lacks support for the {py:obj}`~fastcs.datatypes.table.Table` type, -which is used to implement structs and tuples. +Workaround: Use {py:obj}`fastcs_secop.SecopQuirks.skip_accessibles` to skip the accessible, or use +{py:obj}`fastcs_secop.SecopQuirks.raw_accessibles` to read/write to the accessible in +'raw' mode, which treats the SECoP JSON value as a string. -Workaround: Use {py:obj}`fastcs_secop.SecopQuirks` to either skip the accessible, or read it in -'raw' mode, which treats the SECoP JSON response as a string to be interpreted downstream, rather than deserialising it. -A {py:obj}`~collections.defaultdict` can be used to specify reading *all* arrays, structs or tuples in raw mode. +You can also use {py:obj}`fastcs_secop.SecopQuirks.raw_tuple` / {py:obj}`~fastcs_secop.SecopQuirks.raw_struct` +/ {py:obj}`~fastcs_secop.SecopQuirks.raw_array` to unconditionally read any tuple/struct/array channel in raw mode. +{#limitations_async} ## Asynchronous updates Asynchronous updates are not supported by {py:obj}`fastcs_secop`. They are turned off using a diff --git a/doc/spelling_wordlist.txt b/doc/spelling_wordlist.txt index cef5c9b..2ac1194 100644 --- a/doc/spelling_wordlist.txt +++ b/doc/spelling_wordlist.txt @@ -1,5 +1,6 @@ accessible accessibles +bool datatype datatypes deserialised diff --git a/doc/transports/epics_ca.md b/doc/transports/epics_ca.md new file mode 100644 index 0000000..1823dd3 --- /dev/null +++ b/doc/transports/epics_ca.md @@ -0,0 +1,26 @@ +# EPICS Channel Access + +EPICS CA transport requires `fastcs[epicsca]` to be installed. + +EPICS CA has a maximum length of 40 on parameter descriptions. Set {py:obj}`fastcs_secop.SecopQuirks.max_description_length` to 40 to truncate descriptions. + +## Supported SECoP data types + +EPICS CA transport supports the following {external+secop:doc}`SECoP data types `: +- double +- scaled +- int +- bool +- enum +- string +- blob +- array of double/int/bool/{ref}`enum* `/string + +Other data types can only be read in 'raw' mode. + +## Example CA IOC + +:::python +```{literalinclude} ../../examples/epics_ca.py +``` +::: diff --git a/doc/transports/epics_pva.md b/doc/transports/epics_pva.md new file mode 100644 index 0000000..aed4fef --- /dev/null +++ b/doc/transports/epics_pva.md @@ -0,0 +1,34 @@ +# EPICS PV Access + +EPICS PVA transport requires `fastcs[epicspva]` to be installed. + +## Supported SECoP data types + +EPICS PVA transport supports the following {external+secop:doc}`SECoP data types `: +- double +- scaled +- int +- bool +- enum +- string +- blob +- array of double/int/bool/{ref}`enum* `/string +- tuple of double/int/bool/{ref}`enum* `/string elements +- struct of double/int/bool/{ref}`enum* `/string elements +- matrix + +Other data types can only be read in 'raw' mode. + +## PVI + +{py:obj}`fastcs` exports PVI PVs with the PVA transport. + +SECoP modules can be found under the top-level PVI structure, while SECoP accessibles can be found under +`module_name:PVI`. This means that the IOC is self-describing to downstream clients. + +## Example PVA IOC + +:::python +```{literalinclude} ../../examples/epics_pva.py +``` +::: diff --git a/doc/transports/tango.md b/doc/transports/tango.md new file mode 100644 index 0000000..1720c8f --- /dev/null +++ b/doc/transports/tango.md @@ -0,0 +1,18 @@ +# Tango + +Tango transport requires `fastcs[tango]` to be installed. + +## Supported SECoP data types + +Tango transport supports the following {external+secop:doc}`SECoP data types `: +- double +- scaled +- int +- bool +- enum +- string +- blob +- array of double/int/bool/{ref}`enum* `/string +- matrix (if the matrix has dimensionality <= 2) + +Other data types can only be read in 'raw' mode. diff --git a/examples/epics_ca.py b/examples/epics_ca.py new file mode 100644 index 0000000..89818f7 --- /dev/null +++ b/examples/epics_ca.py @@ -0,0 +1,40 @@ +"""Example CA IOC using :py:obj:`fastcs_secop`.""" + +import asyncio +import logging +import socket + +from fastcs.connections import IPConnectionSettings +from fastcs.launch import FastCS +from fastcs.logging import LogLevel, configure_logging + +from fastcs_secop import SecopQuirks +from fastcs_secop.controllers import SecopController + +if __name__ == "__main__": + from fastcs.transports import EpicsIOCOptions + from fastcs.transports.epics.ca import EpicsCATransport + + configure_logging(level=LogLevel.DEBUG) + logging.basicConfig(level=LogLevel.DEBUG) + + asyncio.get_event_loop().slow_callback_duration = 1000 + + epics_options = EpicsIOCOptions(pv_prefix=f"TE:{socket.gethostname().upper()}:SECOP") + epics_ca = EpicsCATransport(epicsca=epics_options) + + quirks = SecopQuirks(raw_tuple=True, raw_struct=True, max_description_length=40) + + LEWIS = 57677 + DOCKER_GASFLOW = 10801 + + controller = SecopController( + settings=IPConnectionSettings(ip="127.0.0.1", port=LEWIS), + quirks=quirks, + ) + + fastcs = FastCS( + controller, + [epics_ca], + ) + fastcs.run(interactive=True) diff --git a/examples/epics_pva.py b/examples/epics_pva.py new file mode 100644 index 0000000..afc69bf --- /dev/null +++ b/examples/epics_pva.py @@ -0,0 +1,45 @@ +"""Example PVA IOC using :py:obj:`fastcs_secop`.""" + +import asyncio +import logging +import socket + +from fastcs.connections import IPConnectionSettings +from fastcs.launch import FastCS +from fastcs.logging import LogLevel, configure_logging + +from fastcs_secop import SecopQuirks +from fastcs_secop.controllers import SecopController + +if __name__ == "__main__": + from fastcs.transports import EpicsIOCOptions + from fastcs.transports.epics.pva import EpicsPVATransport + + configure_logging(level=LogLevel.DEBUG) + logging.basicConfig(level=LogLevel.DEBUG) + + asyncio.get_event_loop().slow_callback_duration = 1000 + + epics_options = EpicsIOCOptions(pv_prefix=f"TE:{socket.gethostname().upper()}:SECOP") + epics_pva = EpicsPVATransport(epicspva=epics_options) + + quirks = SecopQuirks( + raw_accessibles=[ + ("valve_controller", "_domains_to_extract"), + ("valve_controller", "_terminal_values"), + ], + ) + + LEWIS = 57677 + DOCKER_GASFLOW = 10801 + + controller = SecopController( + settings=IPConnectionSettings(ip="127.0.0.1", port=LEWIS), + quirks=quirks, + ) + + fastcs = FastCS( + controller, + [epics_pva], + ) + fastcs.run(interactive=True) diff --git a/pyproject.toml b/pyproject.toml index 9ddf14a..4c88272 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -7,7 +7,7 @@ name = "fastcs-secop" # REQUIRED, is the only field that cannot be marked as dy dynamic = ["version"] description = "SECoP device support using FastCS" readme = "README.md" -requires-python = ">=3.11" +requires-python = ">=3.12" license-files = ["LICENSE"] authors = [ @@ -18,22 +18,16 @@ maintainers = [ ] classifiers = [ - # How mature is this project? Common values are - # 3 - Alpha - # 4 - Beta - # 5 - Production/Stable "Development Status :: 3 - Alpha", "Intended Audience :: Developers", "Programming Language :: Python :: 3", - "Programming Language :: Python :: 3.11", "Programming Language :: Python :: 3.12", - "Programming Language :: Python :: 3.13", "Programming Language :: Python :: 3 :: Only", ] dependencies = [ - "fastcs[epics] @ git+https://github.com/Tom-Willemsen/FastCS@guard_methods_that_dont_work_on_windows", + "fastcs @ git+https://github.com/Tom-Willemsen/FastCS@guard_methods_that_dont_work_on_windows", ] [project.optional-dependencies] @@ -88,7 +82,7 @@ exclude_lines = [ directory = "coverage_html_report" [tool.pyright] -include = ["src"] +include = ["src", "examples"] reportConstantRedefinition = true reportDeprecated = true reportInconsistentConstructor = true diff --git a/src/fastcs_secop/__init__.py b/src/fastcs_secop/__init__.py index b8a2316..403f068 100644 --- a/src/fastcs_secop/__init__.py +++ b/src/fastcs_secop/__init__.py @@ -1,49 +1,5 @@ """SECoP support using FastCS.""" -import asyncio -import logging -from collections import defaultdict -from logging import getLogger - -from fastcs.connections import IPConnectionSettings -from fastcs.launch import FastCS -from fastcs.logging import LogLevel, configure_logging -from fastcs.transports import EpicsIOCOptions, EpicsPVATransport -from fastcs.transports.epics.ca import EpicsCATransport - from fastcs_secop._util import SecopError, SecopQuirks -from fastcs_secop.controllers import SecopController - -logger = getLogger(__name__) __all__ = ["SecopError", "SecopQuirks"] - - -if __name__ == "__main__": # pragma: no cover - configure_logging(level=LogLevel.DEBUG) - - logging.basicConfig(level=LogLevel.DEBUG) - - asyncio.get_event_loop().slow_callback_duration = 1000 - - epics_options = EpicsIOCOptions(pv_prefix="TE:NDW2922:SECOP") - epics_ca = EpicsCATransport(epicsca=epics_options) - epics_pva = EpicsPVATransport(epicspva=epics_options) - - quirks = defaultdict( - lambda: SecopQuirks(raw_tuple=False, raw_struct=False, max_description_length=40) - ) - quirks["valve_controller._domains_to_extract"] = SecopQuirks(raw_array=True) - quirks["valve_controller._terminal_values"] = SecopQuirks(raw_struct=True) - - LEWIS = 57677 - DOCKER_GASFLOW = 10801 - - fastcs = FastCS( - SecopController( - settings=IPConnectionSettings(ip="127.0.0.1", port=LEWIS), - quirks=quirks, - ), - [epics_pva], - ) - fastcs.run(interactive=True) diff --git a/src/fastcs_secop/_util.py b/src/fastcs_secop/_util.py index c08d560..75327a4 100644 --- a/src/fastcs_secop/_util.py +++ b/src/fastcs_secop/_util.py @@ -1,4 +1,5 @@ -from dataclasses import dataclass +from collections.abc import Collection +from dataclasses import dataclass, field from typing import Any import numpy as np @@ -7,11 +8,48 @@ @dataclass(frozen=True) class SecopQuirks: - skip: bool = False + """Define special handling for SECoP modules or accessibles. + + Not all combinations of SECoP features can be handled by all + transports. :py:obj:`SecopQuirks` allows specifying non-default + behaviour to work around these limitations. + """ + + update_period: float = 1.0 + """Update period, in seconds, for this accessible.""" + + skip_modules: Collection[str] = field(default_factory=list) + """Skip creating any listed modules.""" + + skip_accessibles: Collection[tuple[str, str]] = field(default_factory=list) + """Skip creating any listed (module_name, accessible_name) tuples.""" + + raw_accessibles: Collection[tuple[str, str]] = field(default_factory=list) + """Create any listed (module_name, accessible_name) tuples in 'raw' mode.""" + raw_array: bool = False + """If the accessible has an array type, read it in raw mode. + + This is useful for accessibles which contain unsupportable arrays + (e.g. nested, potentially ragged, arrays). + """ raw_tuple: bool = False + """If the accessible has a tuple type, read it in raw mode. + + This is useful for transports which do not support the FastCS + :py:obj:`~fastcs.datatypes.table.Table` type. + """ raw_struct: bool = False + """If the accessible has a struct type, read it in raw mode. + + This is useful for transports which do not support the FastCS + :py:obj:`~fastcs.datatypes.table.Table` type. + """ max_description_length: int | None = None + """Truncate accessible descriptions to this length. + + This is useful for transports such as EPICS CA which have a maximum description length. + """ class SecopError(Exception): diff --git a/src/fastcs_secop/controllers.py b/src/fastcs_secop/controllers.py index 40ea20a..15771a1 100644 --- a/src/fastcs_secop/controllers.py +++ b/src/fastcs_secop/controllers.py @@ -36,7 +36,7 @@ def __init__( connection: IPConnection, module_name: str, module: dict[str, typing.Any], - quirks: typing.Mapping[str, SecopQuirks], + quirks: SecopQuirks, ) -> None: """FastCS controller for a SECoP module. @@ -49,8 +49,8 @@ def __init__( module: A deserialised description, in the :external+secop:doc:`SECoP over-the-wire format `, of this module. - quirks: dict-like object of :py:obj:`~fastcs_secop.SecopQuirks` that affects how - attributes are processed. + quirks: Affects how attributes are processed. + See :py:obj:`~fastcs_secop.SecopQuirks` for details. """ self._module_name = module_name @@ -66,9 +66,7 @@ def __init__( async def initialise(self) -> None: # noqa PLR0912 TODO """Create attributes for all accessibles in this SECoP module.""" for parameter_name, parameter in self._module["accessibles"].items(): - quirks = self._quirks.get(f"{self._module_name}.{parameter_name}", SecopQuirks()) - - if quirks.skip: + if (self._module_name, parameter_name) in self._quirks.skip_accessibles: continue logger.debug("Creating attribute for parameter %s", parameter_name) @@ -79,17 +77,23 @@ async def initialise(self) -> None: # noqa PLR0912 TODO max_val = datainfo.get("max") scale = datainfo.get("scale") - description = parameter.get("description", "")[: quirks.max_description_length] + description = parameter.get("description", "")[: self._quirks.max_description_length] attr_cls = AttrR if parameter.get("readonly", False) else AttrRW io_ref = SecopAttributeIORef( module_name=self._module_name, accessible_name=parameter_name, - update_period=1.0, + update_period=self._quirks.update_period, datainfo=datainfo, ) + raw_io_ref = SecopRawAttributeIORef( + module_name=self._module_name, + accessible_name=parameter_name, + update_period=self._quirks.update_period, + ) + if secop_dtype == "command": # TODO: handle commands pass @@ -164,18 +168,15 @@ async def initialise(self) -> None: # noqa PLR0912 TODO ), ) elif secop_dtype == "array": - if self._quirks.get( - f"{self._module_name}.{parameter_name}", SecopQuirks() - ).raw_array: + if ( + self._quirks.raw_array + or (self._module_name, parameter_name) in self._quirks.raw_accessibles + ): self.add_attribute( parameter_name, attr_cls( String(65536), - io_ref=SecopRawAttributeIORef( - module_name=self._module_name, - accessible_name=parameter_name, - update_period=1.0, - ), + io_ref=raw_io_ref, description=description, ), ) @@ -192,18 +193,15 @@ async def initialise(self) -> None: # noqa PLR0912 TODO ), ) elif secop_dtype == "tuple": - if self._quirks.get( - f"{self._module_name}.{parameter_name}", SecopQuirks() - ).raw_tuple: + if ( + self._quirks.raw_tuple + or (self._module_name, parameter_name) in self._quirks.raw_accessibles + ): self.add_attribute( parameter_name, attr_cls( String(65536), - io_ref=SecopRawAttributeIORef( - module_name=self._module_name, - accessible_name=parameter_name, - update_period=1.0, - ), + io_ref=raw_io_ref, description=description, ), ) @@ -219,18 +217,15 @@ async def initialise(self) -> None: # noqa PLR0912 TODO ), ) elif secop_dtype == "struct": - if self._quirks.get( - f"{self._module_name}.{parameter_name}", SecopQuirks() - ).raw_struct: + if ( + self._quirks.raw_struct + or (self._module_name, parameter_name) in self._quirks.raw_accessibles + ): self.add_attribute( parameter_name, attr_cls( String(65536), - io_ref=SecopRawAttributeIORef( - module_name=self._module_name, - accessible_name=parameter_name, - update_period=1.0, - ), + io_ref=raw_io_ref, description=description, ), ) @@ -252,9 +247,7 @@ async def initialise(self) -> None: # noqa PLR0912 TODO class SecopController(Controller): """FastCS Controller for a SECoP node.""" - def __init__( - self, settings: IPConnectionSettings, quirks: typing.Mapping[str, SecopQuirks] | None = None - ) -> None: + def __init__(self, settings: IPConnectionSettings, quirks: SecopQuirks | None = None) -> None: """FastCS Controller for a SECoP node. Args: @@ -263,16 +256,10 @@ def __init__( quirks: :py:obj:`dict`-like object of :py:obj:`~fastcs_secop.SecopQuirks` that affects how attributes are processed. - Quirks may be applied to a module, keyed by ``module_name``, or to an - individual parameter, keyed by ``module_name.parameter_name``. - - Hint: use a :py:obj:`~collections.defaultdict` to - specify quirks that apply to all attributes and modules. - """ self._ip_settings = settings self._connection = IPConnection() - self.quirks = quirks if quirks is not None else {} + self.quirks = quirks or SecopQuirks() super().__init__() @@ -376,7 +363,7 @@ async def initialise(self) -> None: modules = descriptor["modules"] for module_name, module in modules.items(): - if self.quirks.get(module_name, SecopQuirks()).skip: + if module_name in self.quirks.skip_modules: continue logger.debug("Creating subcontroller for module %s", module_name) module_controller = SecopModuleController( diff --git a/src/fastcs_secop/io.py b/src/fastcs_secop/io.py index d67d725..f68786e 100644 --- a/src/fastcs_secop/io.py +++ b/src/fastcs_secop/io.py @@ -1,6 +1,7 @@ """Implementation of IO for SECoP accessibles.""" import base64 +import enum import json from dataclasses import dataclass, field from enum import Enum @@ -22,7 +23,7 @@ logger = getLogger(__name__) -T: TypeAlias = int | float | str | bool | Enum | npt.NDArray[Any] +T: TypeAlias = int | float | str | bool | Enum | npt.NDArray[Any] # noqa: UP040 (sphinx doesn't like it) """Generic type parameter for SECoP IO.""" @@ -149,16 +150,22 @@ def encode(self, value: T, datainfo: dict[str, Any]) -> str: """ match datainfo["type"]: - case "int" | "bool" | "double" | "string" | "enum": + case "int" | "bool" | "double" | "string": return json.dumps(value) + case "enum": + assert isinstance(value, enum.Enum) + return json.dumps(value.value) case "scaled": return json.dumps(round(value / datainfo["scale"])) case "blob": assert isinstance(value, np.ndarray) return json.dumps(base64.b64encode("".join(chr(c) for c in value).encode("utf-8"))) - case "array" | "tuple": + case "array": assert isinstance(value, np.ndarray) return json.dumps(value.tolist()) + case "tuple" | "struct": + assert isinstance(value, np.ndarray) + return json.dumps(value.tolist()[0]) case _: raise SecopError(f"Cannot handle SECoP dtype '{datainfo['type']}'") @@ -186,6 +193,8 @@ async def send(self, attr: AttrW[T, SecopAttributeIORef], value: T) -> None: attr.io_ref.accessible_name, encoded_value, ) + # Ugly, but I can't find a public alternative... + await attr._call_sync_setpoint_callbacks(value) # noqa: SLF001 except ConnectionError: # Reconnect will be attempted in a periodic scan task pass @@ -216,7 +225,9 @@ async def update(self, attr: AttrR[str, SecopRawAttributeIORef]) -> None: raw_value = await secop_read( self._connection, attr.io_ref.module_name, attr.io_ref.accessible_name ) - await attr.update(raw_value) + # Get rid of timestamp and other specifiers, we just want the value + value, *_ = json.loads(raw_value) + await attr.update(json.dumps(value)) except ConnectionError: # Reconnect will be attempted in a periodic scan task pass @@ -232,6 +243,8 @@ async def send(self, attr: AttrW[str, SecopRawAttributeIORef], value: str) -> No attr.io_ref.accessible_name, value, ) + # Ugly, but I can't find a public alternative... + await attr._call_sync_setpoint_callbacks(value) # noqa: SLF001 except ConnectionError: # Reconnect will be attempted in a periodic scan task pass From 2ba71b65a48b9e644d3ba278d0ecd678fa559ba3 Mon Sep 17 00:00:00 2001 From: Tom Willemsen Date: Mon, 29 Dec 2025 19:48:02 +0000 Subject: [PATCH 13/19] Support commands --- doc/spelling_wordlist.txt | 2 + doc/transports/epics_ca.md | 1 + doc/transports/epics_pva.md | 1 + doc/transports/tango.md | 1 + examples/epics_ca.py | 12 +- examples/epics_pva.py | 2 +- pyproject.toml | 1 + ruff.toml | 2 + src/fastcs_secop/_util.py | 62 ++++++- src/fastcs_secop/controllers.py | 291 ++++++++++++++------------------ src/fastcs_secop/io.py | 148 ++++++++-------- tests/test_controller.py | 40 ----- tests/test_util.py | 44 +++++ 13 files changed, 326 insertions(+), 281 deletions(-) create mode 100644 tests/test_util.py diff --git a/doc/spelling_wordlist.txt b/doc/spelling_wordlist.txt index 2ac1194..43996ac 100644 --- a/doc/spelling_wordlist.txt +++ b/doc/spelling_wordlist.txt @@ -1,8 +1,10 @@ accessible accessibles bool +datainfo datatype datatypes +deserialisation deserialised deserialising enum diff --git a/doc/transports/epics_ca.md b/doc/transports/epics_ca.md index 1823dd3..b0eebac 100644 --- a/doc/transports/epics_ca.md +++ b/doc/transports/epics_ca.md @@ -15,6 +15,7 @@ EPICS CA transport supports the following {external+secop:doc}`SECoP data types - string - blob - array of double/int/bool/{ref}`enum* `/string +- command (if arguments and return values are empty or one of the above types) Other data types can only be read in 'raw' mode. diff --git a/doc/transports/epics_pva.md b/doc/transports/epics_pva.md index aed4fef..f781b25 100644 --- a/doc/transports/epics_pva.md +++ b/doc/transports/epics_pva.md @@ -16,6 +16,7 @@ EPICS PVA transport supports the following {external+secop:doc}`SECoP data types - tuple of double/int/bool/{ref}`enum* `/string elements - struct of double/int/bool/{ref}`enum* `/string elements - matrix +- command (if arguments and return values are empty or one of the above types) Other data types can only be read in 'raw' mode. diff --git a/doc/transports/tango.md b/doc/transports/tango.md index 1720c8f..3611ec9 100644 --- a/doc/transports/tango.md +++ b/doc/transports/tango.md @@ -14,5 +14,6 @@ Tango transport supports the following {external+secop:doc}`SECoP data types `/string - matrix (if the matrix has dimensionality <= 2) +- command (if arguments and return values are empty or one of the above types) Other data types can only be read in 'raw' mode. diff --git a/examples/epics_ca.py b/examples/epics_ca.py index 89818f7..14732d5 100644 --- a/examples/epics_ca.py +++ b/examples/epics_ca.py @@ -23,13 +23,21 @@ epics_options = EpicsIOCOptions(pv_prefix=f"TE:{socket.gethostname().upper()}:SECOP") epics_ca = EpicsCATransport(epicsca=epics_options) - quirks = SecopQuirks(raw_tuple=True, raw_struct=True, max_description_length=40) + quirks = SecopQuirks( + raw_tuple=True, + raw_struct=True, + max_description_length=40, + raw_accessibles=[ + ("valve_controller", "_domains_to_extract"), + ("valve_controller", "_terminal_values"), + ], + ) LEWIS = 57677 DOCKER_GASFLOW = 10801 controller = SecopController( - settings=IPConnectionSettings(ip="127.0.0.1", port=LEWIS), + settings=IPConnectionSettings(ip="127.0.0.1", port=DOCKER_GASFLOW), quirks=quirks, ) diff --git a/examples/epics_pva.py b/examples/epics_pva.py index afc69bf..e7e4702 100644 --- a/examples/epics_pva.py +++ b/examples/epics_pva.py @@ -34,7 +34,7 @@ DOCKER_GASFLOW = 10801 controller = SecopController( - settings=IPConnectionSettings(ip="127.0.0.1", port=LEWIS), + settings=IPConnectionSettings(ip="127.0.0.1", port=DOCKER_GASFLOW), quirks=quirks, ) diff --git a/pyproject.toml b/pyproject.toml index 4c88272..a24caf5 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -28,6 +28,7 @@ classifiers = [ dependencies = [ "fastcs @ git+https://github.com/Tom-Willemsen/FastCS@guard_methods_that_dont_work_on_windows", + "orjson", ] [project.optional-dependencies] diff --git a/ruff.toml b/ruff.toml index 3aa4957..112a5c6 100644 --- a/ruff.toml +++ b/ruff.toml @@ -51,3 +51,5 @@ ignore = [ [lint.pylint] max-args = 8 +max-returns = 10 +max-branches = 15 diff --git a/src/fastcs_secop/_util.py b/src/fastcs_secop/_util.py index 75327a4..a254365 100644 --- a/src/fastcs_secop/_util.py +++ b/src/fastcs_secop/_util.py @@ -1,9 +1,11 @@ +import enum from collections.abc import Collection from dataclasses import dataclass, field from typing import Any import numpy as np import numpy.typing as npt +from fastcs.datatypes import Bool, DataType, Enum, Float, Int, String, Table, Waveform @dataclass(frozen=True) @@ -74,7 +76,7 @@ def secop_dtype_to_numpy_dtype(secop_datainfo: dict[str, Any]) -> npt.DTypeLike: elif dtype == "int": return np.int32 elif dtype == "bool": - return np.uint8 + return np.uint8 # CA transport doesn't support bool_ elif dtype == "enum": return np.int32 elif dtype == "string": @@ -98,3 +100,61 @@ def struct_structured_dtype(datainfo: dict[str, Any]) -> list[tuple[str, npt.DTy (k, secop_dtype_to_numpy_dtype(v)) for k, v in datainfo["members"].items() ] return structured_np_dtype + + +def secop_datainfo_to_fastcs_dtype(datainfo: dict[str, Any], raw: bool = False) -> DataType[Any]: + """Convert a SECoP datainfo dictionary to a FastCS data type. + + Args: + datainfo: SECoP datainfo dictionary. + raw: whether to read this parameter in 'raw' mode. + + """ + if raw: + return String() + + min_val = datainfo.get("min") + max_val = datainfo.get("max") + + match datainfo["type"]: + case "double" | "scaled": + scale = datainfo.get("scale") + + if min_val is not None and scale is not None: + min_val *= scale + if max_val is not None and scale is not None: + max_val *= scale + + return Float( + units=datainfo.get("unit", None), + min_alarm=min_val, + max_alarm=max_val, + prec=format_string_to_prec(datainfo.get("fmtstr", None)), # type: ignore + ) + case "int": + return Int( + units=datainfo.get("unit", None), + min_alarm=min_val, + max_alarm=max_val, + ) + case "bool": + return Bool() + case "enum": + enum_type = enum.Enum("enum_type", datainfo["members"]) + return Enum(enum_type) + case "string": + return String() + case "blob": + return Waveform(np.uint8, shape=(datainfo["maxbytes"],)) + case "array": + inner_dtype = datainfo["members"] + np_inner_dtype = secop_dtype_to_numpy_dtype(inner_dtype) + return Waveform(np_inner_dtype, shape=(datainfo["maxlen"],)) + case "tuple": + structured_dtype = tuple_structured_dtype(datainfo) + return Table(structured_dtype) + case "struct": + structured_dtype = struct_structured_dtype(datainfo) + return Table(structured_dtype) + case _: + raise SecopError(f"Invalid SECoP dtype for FastCS attribute: {datainfo['type']}") diff --git a/src/fastcs_secop/controllers.py b/src/fastcs_secop/controllers.py index 15771a1..8a1e42b 100644 --- a/src/fastcs_secop/controllers.py +++ b/src/fastcs_secop/controllers.py @@ -1,32 +1,110 @@ """FastCS controllers for SECoP nodes.""" -import enum -import json import typing import uuid from logging import getLogger -import numpy as np +import orjson from fastcs.attributes import AttrR, AttrRW from fastcs.connections import IPConnection, IPConnectionSettings from fastcs.controllers import Controller -from fastcs.datatypes import Bool, Enum, Float, Int, String, Table, Waveform -from fastcs.methods import scan +from fastcs.methods import command, scan from fastcs_secop import SecopQuirks -from fastcs_secop._util import SecopError, format_string_to_prec, struct_structured_dtype +from fastcs_secop._util import ( + SecopError, + secop_datainfo_to_fastcs_dtype, +) from fastcs_secop.io import ( SecopAttributeIO, SecopAttributeIORef, SecopRawAttributeIO, SecopRawAttributeIORef, - secop_dtype_to_numpy_dtype, - tuple_structured_dtype, + decode, + encode, ) logger = getLogger(__name__) +class SecopCommandController(Controller): + """SECoP command controller.""" + + def __init__( + self, + *, + connection: IPConnection, + module_name: str, + command_name: str, + datainfo: dict[str, typing.Any], + ) -> None: + """Subcontroller for a SECoP command. + + Args: + connection: The connection to use. + module_name: The module in which this command is defined. + command_name: The name of the command. + datainfo: The datainfo dictionary for this command. + + """ + super().__init__() + + self.connection = connection + self.module_name = module_name + self.command_name = command_name + self.datainfo = datainfo + + async def initialise(self) -> None: + """Initialise the command controller. + + This will set up PVs for ``Args`` and ``Result`` (if they have a type). + """ + if self.datainfo.get("argument") is not None: + args_type = secop_datainfo_to_fastcs_dtype(self.datainfo["argument"]) + else: + args_type = None + + if self.datainfo.get("result") is not None: + result_type = secop_datainfo_to_fastcs_dtype(self.datainfo["result"]) + else: + result_type = None + + if args_type is not None: + self.args = AttrRW(description="args", datatype=args_type) + else: + self.args = None + + if result_type is not None: + self.result = AttrR(description="result", datatype=result_type) + else: + self.result = None + + @command() + async def execute(self) -> None: + """Execute the command.""" + prefix = f"do {self.module_name}:{self.command_name}" + response_prefix = f"done {self.module_name}:{self.command_name}" + + if self.args is not None: + cmd = f"{prefix} {encode(self.args.get(), self.datainfo['argument'])}\n" + else: + cmd = f"{prefix}\n" + + logger.debug("Sending command: '%s'", cmd) + response = await self.connection.send_query(cmd) + logger.debug("Response: '%s'", response) + + response = response.strip() + if not response.startswith(response_prefix): + logger.warning("command '%s' failed (response='%s')", prefix, response) + return + + response = response[len(response_prefix) :].strip() + + if self.result is not None: + await self.result.update(decode(response, self.datainfo["result"], self.result)) + + class SecopModuleController(Controller): """FastCS controller for a SECoP module.""" @@ -56,6 +134,8 @@ def __init__( self._module_name = module_name self._module = module self._quirks = quirks + self._connection = connection + super().__init__( ios=[ SecopAttributeIO(connection=connection), @@ -63,7 +143,15 @@ def __init__( ] ) - async def initialise(self) -> None: # noqa PLR0912 TODO + def _is_raw(self, parameter_name: str, datainfo: dict[str, typing.Any]) -> bool: + return ( + ((self._module_name, parameter_name) in self._quirks.raw_accessibles) + or (datainfo["type"] == "array" and self._quirks.raw_array) + or (datainfo["type"] == "tuple" and self._quirks.raw_tuple) + or (datainfo["type"] == "struct" and self._quirks.raw_struct) + ) + + async def initialise(self) -> None: """Create attributes for all accessibles in this SECoP module.""" for parameter_name, parameter in self._module["accessibles"].items(): if (self._module_name, parameter_name) in self._quirks.skip_accessibles: @@ -71,177 +159,47 @@ async def initialise(self) -> None: # noqa PLR0912 TODO logger.debug("Creating attribute for parameter %s", parameter_name) datainfo = parameter["datainfo"] - secop_dtype = datainfo["type"] - - min_val = datainfo.get("min") - max_val = datainfo.get("max") - scale = datainfo.get("scale") description = parameter.get("description", "")[: self._quirks.max_description_length] attr_cls = AttrR if parameter.get("readonly", False) else AttrRW - io_ref = SecopAttributeIORef( - module_name=self._module_name, - accessible_name=parameter_name, - update_period=self._quirks.update_period, - datainfo=datainfo, - ) - - raw_io_ref = SecopRawAttributeIORef( - module_name=self._module_name, - accessible_name=parameter_name, - update_period=self._quirks.update_period, - ) + raw = self._is_raw(parameter_name, datainfo) - if secop_dtype == "command": - # TODO: handle commands - pass - elif secop_dtype in {"double", "scaled"}: - if min_val is not None and scale is not None: - min_val *= scale - if max_val is not None and scale is not None: - max_val *= scale - - self.add_attribute( - parameter_name, - attr_cls( - Float( - units=datainfo.get("unit", None), - min_alarm=min_val, - max_alarm=max_val, - prec=format_string_to_prec(datainfo.get("fmtstr", None)) or 6, - ), - io_ref=io_ref, - description=description, - ), + if raw: + io_ref = SecopRawAttributeIORef( + module_name=self._module_name, + accessible_name=parameter_name, + update_period=self._quirks.update_period, ) - elif secop_dtype == "int": - self.add_attribute( - parameter_name, - attr_cls( - Int( - units=datainfo.get("unit", None), - min_alarm=min_val, - max_alarm=max_val, - ), - io_ref=io_ref, - description=description, - ), - ) - elif secop_dtype == "bool": - self.add_attribute( - parameter_name, - attr_cls( - Bool(), - io_ref=io_ref, - description=description, - ), + else: + io_ref = SecopAttributeIORef( + module_name=self._module_name, + accessible_name=parameter_name, + update_period=self._quirks.update_period, + datainfo=datainfo, ) - elif secop_dtype == "enum": - enum_type = enum.Enum("enum_type", datainfo["members"]) - self.add_attribute( - parameter_name, - attr_cls( - Enum(enum_type), - io_ref=io_ref, - description=description, - ), - ) - elif secop_dtype == "string": - self.add_attribute( - parameter_name, - attr_cls( - String(), - io_ref=io_ref, - description=description, - ), + if datainfo["type"] == "command": + command_controller = SecopCommandController( + module_name=self._module_name, + command_name=parameter_name, + connection=self._connection, + datainfo=datainfo, ) - elif secop_dtype == "blob": + self.add_sub_controller(parameter_name, command_controller) + await command_controller.initialise() + else: + fastcs_type = secop_datainfo_to_fastcs_dtype(datainfo=datainfo, raw=raw) + self.add_attribute( parameter_name, attr_cls( - Waveform(np.uint8, shape=(datainfo["maxbytes"],)), + fastcs_type, io_ref=io_ref, description=description, ), ) - elif secop_dtype == "array": - if ( - self._quirks.raw_array - or (self._module_name, parameter_name) in self._quirks.raw_accessibles - ): - self.add_attribute( - parameter_name, - attr_cls( - String(65536), - io_ref=raw_io_ref, - description=description, - ), - ) - else: - inner_dtype = datainfo["members"] - np_inner_dtype = secop_dtype_to_numpy_dtype(inner_dtype) - - self.add_attribute( - parameter_name, - attr_cls( - Waveform(np_inner_dtype, shape=(datainfo["maxlen"],)), - io_ref=io_ref, - description=description, - ), - ) - elif secop_dtype == "tuple": - if ( - self._quirks.raw_tuple - or (self._module_name, parameter_name) in self._quirks.raw_accessibles - ): - self.add_attribute( - parameter_name, - attr_cls( - String(65536), - io_ref=raw_io_ref, - description=description, - ), - ) - else: - structured_dtype = tuple_structured_dtype(datainfo) - - self.add_attribute( - parameter_name, - attr_cls( - Table(structured_dtype), - io_ref=io_ref, - description=description, - ), - ) - elif secop_dtype == "struct": - if ( - self._quirks.raw_struct - or (self._module_name, parameter_name) in self._quirks.raw_accessibles - ): - self.add_attribute( - parameter_name, - attr_cls( - String(65536), - io_ref=raw_io_ref, - description=description, - ), - ) - else: - structured_dtype = struct_structured_dtype(datainfo) - - self.add_attribute( - parameter_name, - attr_cls( - Table(structured_dtype), - io_ref=io_ref, - description=description, - ), - ) - else: - raise SecopError(f"Unsupported secop data type '{secop_dtype}") class SecopController(Controller): @@ -253,8 +211,7 @@ def __init__(self, settings: IPConnectionSettings, quirks: SecopQuirks | None = Args: settings: The communication settings (e.g. IP address, port) at which the SECoP node is reachable. - quirks: :py:obj:`dict`-like object of :py:obj:`~fastcs_secop.SecopQuirks` - that affects how attributes are processed. + quirks: :py:obj:`~fastcs_secop.SecopQuirks` that affects how attributes are processed. """ self._ip_settings = settings @@ -352,13 +309,13 @@ async def initialise(self) -> None: if not descriptor.startswith("describing . "): raise SecopError(f"Invalid response to 'describe': '{descriptor}'.") - descriptor = json.loads(descriptor[len("describing . ") :]) + descriptor = orjson.loads(descriptor[len("describing . ") :]) description = descriptor["description"] equipment_id = descriptor["equipment_id"] logger.info("SECoP equipment_id = '%s', description = '%s'", equipment_id, description) - logger.debug("descriptor = %s", json.dumps(descriptor, indent=2)) + logger.debug("descriptor = %s", orjson.dumps(descriptor, option=orjson.OPT_INDENT_2)) modules = descriptor["modules"] diff --git a/src/fastcs_secop/io.py b/src/fastcs_secop/io.py index f68786e..c479fbc 100644 --- a/src/fastcs_secop/io.py +++ b/src/fastcs_secop/io.py @@ -2,7 +2,6 @@ import base64 import enum -import json from dataclasses import dataclass, field from enum import Enum from logging import getLogger @@ -10,6 +9,7 @@ import numpy as np import numpy.typing as npt +import orjson from fastcs.attributes import AttributeIO, AttributeIORef, AttrR, AttrW from fastcs.connections import IPConnection @@ -36,7 +36,7 @@ async def secop_read(connection: IPConnection, module_name: str, accessible_name accessible_name: Accessible name Returns: - The result of reading from the accessible, after calling :py:obj:`json.loads`. + The result of reading from the accessible, after JSON deserialisation. Raises: SecopError: If a valid response was not received @@ -96,6 +96,78 @@ class SecopRawAttributeIORef(AttributeIORef): accessible_name: str = "" +def decode(value: str, datainfo: dict[str, Any], attr: AttrR[T]) -> T: # noqa ANN401 + """Decode the transported value into a python datatype. + + Args: + value: The value to decode (the raw transported string) + datainfo: The SECoP ``datainfo`` dictionary for this attribute. + + Returns: + Python datatype representation of the transported value. + + """ + value, *_ = orjson.loads(value) + match datainfo["type"]: + case "enum": + return attr.dtype(cast(int, value)) + case "scaled": + return value * datainfo["scale"] + case "blob": + return np.frombuffer(base64.b64decode(value), dtype=np.uint8) + case "array": + inner_np_dtype = secop_dtype_to_numpy_dtype(datainfo["members"]) + return np.array(value, dtype=inner_np_dtype) + case "tuple": + structured_np_dtype = tuple_structured_dtype(datainfo) + return np.array([tuple(value)], dtype=structured_np_dtype) + case "struct": + structured_np_dtype = struct_structured_dtype(datainfo) + arr = np.zeros(shape=(1,), dtype=structured_np_dtype) + for k, v in cast(dict[str, Any], value).items(): + arr[0][k] = v + return arr + case _: + return value + + +def encode(value: T, datainfo: dict[str, Any]) -> str: + """Encode the transported value to a string for transport. + + Args: + value: The value to encode. + datainfo: The SECoP ``datainfo`` dictionary for this attribute. + + """ + match datainfo["type"]: + case "int" | "bool" | "double" | "string": + return orjson.dumps(value).decode() + case "enum": + assert isinstance(value, enum.Enum) + return orjson.dumps(value.value).decode() + case "scaled": + return orjson.dumps(round(value / datainfo["scale"])).decode() + case "blob": + assert isinstance(value, np.ndarray) + return orjson.dumps( + base64.b64encode("".join(chr(c) for c in value).encode("utf-8")) + ).decode() + case "array": + return orjson.dumps(value, option=orjson.OPT_SERIALIZE_NUMPY).decode() + case "tuple": + assert isinstance(value, np.ndarray) + return orjson.dumps(value[0], option=orjson.OPT_SERIALIZE_NUMPY).decode() + case "struct": + assert isinstance(value, np.ndarray) + ans = {} + assert value.dtype.names is not None + for name in value.dtype.names: + ans[name] = value[name][0] + return orjson.dumps(ans, option=orjson.OPT_SERIALIZE_NUMPY).decode() + case _: + raise SecopError(f"Cannot handle SECoP dtype '{datainfo['type']}'") + + class SecopAttributeIO(AttributeIO[T, SecopAttributeIORef]): """IO for a SECoP parameter of any type other than 'command'.""" @@ -105,77 +177,13 @@ def __init__(self, *, connection: IPConnection) -> None: self._connection = connection - def decode(self, value: str, datainfo: dict[str, Any], attr: AttrR[T]) -> T: # noqa ANN401 - """Decode the transported value into a python datatype. - - Args: - value: The value to decode (the raw transported string) - datainfo: The SECoP ``datainfo`` dictionary for this attribute. - - Returns: - Python datatype representation of the transported value. - - """ - value, *_ = json.loads(value) - match datainfo["type"]: - case "int" | "bool" | "double" | "string": - return value - case "enum": - return attr.dtype(cast(int, value)) - case "scaled": - return value * datainfo["scale"] - case "blob": - return np.frombuffer(base64.b64decode(value), dtype=np.uint8) - case "array": - inner_np_dtype = secop_dtype_to_numpy_dtype(datainfo["members"]) - return np.array(value, dtype=inner_np_dtype) - case "tuple": - structured_np_dtype = tuple_structured_dtype(datainfo) - return np.array([tuple(value)], dtype=structured_np_dtype) - case "struct": - structured_np_dtype = struct_structured_dtype(datainfo) - arr = np.zeros(shape=(1,), dtype=structured_np_dtype) - for k, v in cast(dict[str, Any], value).items(): - arr[0][k] = v - return arr - case _: - raise SecopError(f"Cannot handle SECoP dtype '{datainfo['type']}'") - - def encode(self, value: T, datainfo: dict[str, Any]) -> str: - """Encode the transported value to a string for transport. - - Args: - value: The value to encode. - datainfo: The SECoP ``datainfo`` dictionary for this attribute. - - """ - match datainfo["type"]: - case "int" | "bool" | "double" | "string": - return json.dumps(value) - case "enum": - assert isinstance(value, enum.Enum) - return json.dumps(value.value) - case "scaled": - return json.dumps(round(value / datainfo["scale"])) - case "blob": - assert isinstance(value, np.ndarray) - return json.dumps(base64.b64encode("".join(chr(c) for c in value).encode("utf-8"))) - case "array": - assert isinstance(value, np.ndarray) - return json.dumps(value.tolist()) - case "tuple" | "struct": - assert isinstance(value, np.ndarray) - return json.dumps(value.tolist()[0]) - case _: - raise SecopError(f"Cannot handle SECoP dtype '{datainfo['type']}'") - async def update(self, attr: AttrR[T, SecopAttributeIORef]) -> None: """Read value from device and update the value in FastCS.""" try: raw_value = await secop_read( self._connection, attr.io_ref.module_name, attr.io_ref.accessible_name ) - value = self.decode(raw_value, attr.io_ref.datainfo, attr) + value = decode(raw_value, attr.io_ref.datainfo, attr) await attr.update(value) except ConnectionError: # Reconnect will be attempted in a periodic scan task @@ -186,7 +194,7 @@ async def update(self, attr: AttrR[T, SecopAttributeIORef]) -> None: async def send(self, attr: AttrW[T, SecopAttributeIORef], value: T) -> None: """Send a value from FastCS to the device.""" try: - encoded_value = self.encode(value, attr.io_ref.datainfo) + encoded_value = encode(value, attr.io_ref.datainfo) await secop_change( self._connection, attr.io_ref.module_name, @@ -226,8 +234,8 @@ async def update(self, attr: AttrR[str, SecopRawAttributeIORef]) -> None: self._connection, attr.io_ref.module_name, attr.io_ref.accessible_name ) # Get rid of timestamp and other specifiers, we just want the value - value, *_ = json.loads(raw_value) - await attr.update(json.dumps(value)) + value, *_ = orjson.loads(raw_value) + await attr.update(orjson.dumps(value).decode()) except ConnectionError: # Reconnect will be attempted in a periodic scan task pass diff --git a/tests/test_controller.py b/tests/test_controller.py index c927f20..e4856e0 100644 --- a/tests/test_controller.py +++ b/tests/test_controller.py @@ -1,14 +1,10 @@ from unittest.mock import AsyncMock, patch -import numpy as np import pytest from fastcs.connections import IPConnectionSettings -from fastcs_secop import SecopError from fastcs_secop.controllers import ( SecopController, - format_string_to_prec, - secop_dtype_to_numpy_dtype, ) @@ -23,42 +19,6 @@ def controller(): return controller -@pytest.mark.parametrize( - ("secop_fmt", "prec"), - [ - ("%.1f", 1), - ("%.99f", 99), - ("%.5g", None), - ("%.5e", None), - (None, None), - ], -) -def test_format_string_to_prec(secop_fmt, prec): - assert format_string_to_prec(secop_fmt) == prec - - -@pytest.mark.parametrize( - ("secop_dtype", "np_dtype"), - [ - ({"type": "int"}, np.int32), - ({"type": "double"}, np.float64), - ({"type": "bool"}, np.uint8), - ({"type": "enum"}, np.int32), - ({"type": "string", "maxchars": 123}, " Date: Fri, 2 Jan 2026 10:55:46 +0000 Subject: [PATCH 14/19] more unit tests --- .github/workflows/Lint-and-test.yml | 2 +- doc/conf.py | 2 + doc/index.md | 8 + doc/limitations.md | 6 + doc/transports/epics_ca.md | 20 +- doc/transports/epics_pva.md | 26 +-- doc/transports/tango.md | 2 +- examples/epics_ca.py | 2 +- pyproject.toml | 15 +- src/fastcs_secop/_util.py | 12 +- src/fastcs_secop/controllers.py | 9 +- src/fastcs_secop/io.py | 8 +- tests/emulators/simple_secop/device.py | 37 +++- tests/test_against_emulator.py | 13 ++ tests/test_controller.py | 141 ++++++++++++++ tests/test_io.py | 243 +++++++++++++++++++++++++ tests/test_util.py | 32 ++++ 17 files changed, 535 insertions(+), 43 deletions(-) create mode 100644 tests/test_io.py diff --git a/.github/workflows/Lint-and-test.yml b/.github/workflows/Lint-and-test.yml index f44f9c5..0a9c4da 100644 --- a/.github/workflows/Lint-and-test.yml +++ b/.github/workflows/Lint-and-test.yml @@ -14,7 +14,7 @@ jobs: with: python-version: ${{ matrix.version }} - name: install requirements - run: uv sync --extra dev + run: uv sync --extra lint --extra test - name: run ruff check run: uv run ruff check - name: run ruff format --check diff --git a/doc/conf.py b/doc/conf.py index c36a77a..73821b4 100644 --- a/doc/conf.py +++ b/doc/conf.py @@ -89,4 +89,6 @@ "secop": ("https://sampleenvironment.github.io/secop-site/", None), "fastcs": ("https://diamondlightsource.github.io/FastCS/main/", None), "numpy": ("https://numpy.org/doc/stable/", None), + "epics": ("https://docs.epics-controls.org/en/latest/", None), + "epics_base": ("https://docs.epics-controls.org/projects/base/en/latest/", None), } diff --git a/doc/index.md b/doc/index.md index 236d423..fcb45e8 100644 --- a/doc/index.md +++ b/doc/index.md @@ -3,6 +3,14 @@ The {py:obj}`fastcs_secop` library implements support for the {external+secop:doc}`SECoP protocol ` using {external+fastcs:doc}`FastCS `. +```{mermaid} +erDiagram + "EPICS Clients" }o--o| "fastcs + fastcs-secop" : "EPICS CA" + "EPICS Clients" }o--o| "fastcs + fastcs-secop" : "EPICS PVA" + "Tango Clients" }o--o| "fastcs + fastcs-secop" : "Tango" + "fastcs + fastcs-secop" ||--|| "SEC Node" : SECoP +``` + ```{toctree} :titlesonly: :caption: Transports diff --git a/doc/limitations.md b/doc/limitations.md index b02eaaa..802b477 100644 --- a/doc/limitations.md +++ b/doc/limitations.md @@ -53,3 +53,9 @@ Asynchronous updates are not supported by {py:obj}`fastcs_secop`. They are turne {external+secop:doc}`deactivate message ` at connection time. Rationale: FastCS does not currently provide infrastructure to handle asynchronous messages. + +{#limitations_qualifiers} +## Timestamp and error qualifiers + +These are ignored; FastCS currently exposes no mechanism to set these. If such a mechanism is later added to FastCS, +they may become supportable here. diff --git a/doc/transports/epics_ca.md b/doc/transports/epics_ca.md index b0eebac..d508a0b 100644 --- a/doc/transports/epics_ca.md +++ b/doc/transports/epics_ca.md @@ -6,18 +6,18 @@ EPICS CA has a maximum length of 40 on parameter descriptions. Set {py:obj}`fast ## Supported SECoP data types -EPICS CA transport supports the following {external+secop:doc}`SECoP data types `: -- double -- scaled -- int -- bool -- enum -- string -- blob -- array of double/int/bool/{ref}`enum* `/string +EPICS CA transport supports the following {external+secop:doc}`SECoP data types `, using the corresponding {external+epics_base:doc}`EPICS record type `: +- double (`ai`/`ao`) +- scaled (`ai`/`ao`) +- int (`longin`/`longout`) +- bool (`bi`/`bo`) +- enum (`mbbi`/`mbbo`) +- string (`lsi`/`lso`) +- blob (`waveform[char]`) +- array of double/int/bool/{ref}`enum* ` (`waveform`) - command (if arguments and return values are empty or one of the above types) -Other data types can only be read in 'raw' mode. +Other data types can only be read or written in 'raw' mode. ## Example CA IOC diff --git a/doc/transports/epics_pva.md b/doc/transports/epics_pva.md index f781b25..83a1561 100644 --- a/doc/transports/epics_pva.md +++ b/doc/transports/epics_pva.md @@ -4,21 +4,21 @@ EPICS PVA transport requires `fastcs[epicspva]` to be installed. ## Supported SECoP data types -EPICS PVA transport supports the following {external+secop:doc}`SECoP data types `: -- double -- scaled -- int -- bool -- enum -- string -- blob -- array of double/int/bool/{ref}`enum* `/string -- tuple of double/int/bool/{ref}`enum* `/string elements -- struct of double/int/bool/{ref}`enum* `/string elements -- matrix +EPICS PVA transport supports the following {external+secop:doc}`SECoP data types ` (using the corresponding {external+epics:doc}`PVA normative types `): +- double (`NTScalar[double]`) +- scaled (`NTScalar[double]`) +- int (`NTScalar[int]`) +- bool (`NTScalar[boolean]`) +- enum (`NTEnum`) +- string (`NTScalar[string]`) +- blob (`NTNDArray[ubyte]`) +- array of double/int/bool/{ref}`enum* ` (`NTNDArray`) +- tuple of double/int/bool/{ref}`enum* `/string elements (`NTTable` with one row) +- struct of double/int/bool/{ref}`enum* `/string elements (`NTTable` with one row) +- matrix (`NTNDArray`) - command (if arguments and return values are empty or one of the above types) -Other data types can only be read in 'raw' mode. +Other data types can only be read or written in 'raw' mode. ## PVI diff --git a/doc/transports/tango.md b/doc/transports/tango.md index 3611ec9..52777a5 100644 --- a/doc/transports/tango.md +++ b/doc/transports/tango.md @@ -16,4 +16,4 @@ Tango transport supports the following {external+secop:doc}`SECoP data types None: identification = await self._connection.send_query("*IDN?\n") identification = identification.strip() - manufacturer, product, _, _ = identification.split(",") + try: + manufacturer, product, _, _ = identification.split(",") + except ValueError as e: + raise SecopError("Invalid response to '*IDN?'") from e + if manufacturer not in { "ISSE&SINE2020", # SECOP 1.x "ISSE", # SECOP 2.x @@ -304,7 +308,10 @@ async def initialise(self) -> None: await self.connect() await self.check_idn() await self.deactivate() + await self.create_modules() + async def create_modules(self) -> None: + """Create subcontrollers for each SECoP module.""" descriptor = await self._connection.send_query("describe\n") if not descriptor.startswith("describing . "): raise SecopError(f"Invalid response to 'describe': '{descriptor}'.") diff --git a/src/fastcs_secop/io.py b/src/fastcs_secop/io.py index c479fbc..ecb39ae 100644 --- a/src/fastcs_secop/io.py +++ b/src/fastcs_secop/io.py @@ -96,7 +96,7 @@ class SecopRawAttributeIORef(AttributeIORef): accessible_name: str = "" -def decode(value: str, datainfo: dict[str, Any], attr: AttrR[T]) -> T: # noqa ANN401 +def decode(raw_value: str, datainfo: dict[str, Any], attr: AttrR[T]) -> T: # noqa ANN401 """Decode the transported value into a python datatype. Args: @@ -107,7 +107,7 @@ def decode(value: str, datainfo: dict[str, Any], attr: AttrR[T]) -> T: # noqa A Python datatype representation of the transported value. """ - value, *_ = orjson.loads(value) + value, *_ = orjson.loads(raw_value) match datainfo["type"]: case "enum": return attr.dtype(cast(int, value)) @@ -150,13 +150,13 @@ def encode(value: T, datainfo: dict[str, Any]) -> str: case "blob": assert isinstance(value, np.ndarray) return orjson.dumps( - base64.b64encode("".join(chr(c) for c in value).encode("utf-8")) + base64.b64encode("".join(chr(c) for c in value).encode("utf-8")).decode() ).decode() case "array": return orjson.dumps(value, option=orjson.OPT_SERIALIZE_NUMPY).decode() case "tuple": assert isinstance(value, np.ndarray) - return orjson.dumps(value[0], option=orjson.OPT_SERIALIZE_NUMPY).decode() + return orjson.dumps(value.tolist()[0]).decode() case "struct": assert isinstance(value, np.ndarray) ans = {} diff --git a/tests/emulators/simple_secop/device.py b/tests/emulators/simple_secop/device.py index 0da9078..026400e 100644 --- a/tests/emulators/simple_secop/device.py +++ b/tests/emulators/simple_secop/device.py @@ -60,11 +60,18 @@ def change(self, value): class Command(Accessible): + def __init__(self, arg_datainfo, result_datainfo): + super().__init__() + self.arg_datainfo = arg_datainfo + self.result_datainfo = result_datainfo + def descriptor(self) -> dict[str, typing.Any]: return { "description": "some_command_description", "datainfo": { "type": "command", + "argument": self.arg_datainfo, + "result": self.result_datainfo, }, } @@ -81,7 +88,7 @@ def __init__(self): prec=4, desc="a scaled parameter", dtype="scaled", - extra_datainfo={"scale": 47}, + extra_datainfo={"scale": 47, "min": 0, "max": 1_000_000}, ), "int": Parameter(73, desc="an integer parameter", dtype="int"), "bool": Parameter(True, desc="a boolean parameter", dtype="bool"), @@ -117,6 +124,18 @@ def __init__(self): dtype="array", extra_datainfo={"maxlen": 512, "members": {"type": "bool"}}, ), + "enum_array": Parameter( + [1, 2, 3, 2, 1], + desc="an enum array parameter", + dtype="array", + extra_datainfo={ + "maxlen": 512, + "members": { + "type": "enum", + "members": {"one": 1, "two": 2, "three": 3, "four": 4, "five": 5}, + }, + }, + ), "tuple": Parameter( [1, 5.678, True, "hiya", 5], desc="a tuple of int, float, bool", @@ -145,6 +164,22 @@ def __init__(self): } }, ), + "command_bool_int": Command( + arg_datainfo={"type": "bool"}, + result_datainfo={"type": "int"}, + ), + "command_null_int": Command( + arg_datainfo=None, + result_datainfo={"type": "int"}, + ), + "command_bool_null": Command( + arg_datainfo={"type": "bool"}, + result_datainfo=None, + ), + "command_null_null": Command( + arg_datainfo=None, + result_datainfo=None, + ), } self.description = "a module with one accessible of each possible dtype" diff --git a/tests/test_against_emulator.py b/tests/test_against_emulator.py index c9bf429..96c7a60 100644 --- a/tests/test_against_emulator.py +++ b/tests/test_against_emulator.py @@ -152,3 +152,16 @@ async def test_attributes_created_for_array_datatype( await attr.wait_for_predicate( lambda v: np.array_equal(v, expected_initial_value), timeout=2 ) + + @pytest.mark.parametrize( + ("command", "expected_attributes"), + [ + ("command_bool_int", {"args", "result"}), + ("command_null_int", {"result"}), + ("command_bool_null", {"args"}), + ("command_null_null", set()), + ], + ) + async def test_command_attributes_created(self, controller, command, expected_attributes): + cmd_controller = controller.sub_controllers["one_of_everything"].sub_controllers[command] + assert set(cmd_controller.attributes.keys()) == expected_attributes diff --git a/tests/test_controller.py b/tests/test_controller.py index e4856e0..70a0306 100644 --- a/tests/test_controller.py +++ b/tests/test_controller.py @@ -1,10 +1,14 @@ from unittest.mock import AsyncMock, patch +import orjson import pytest from fastcs.connections import IPConnectionSettings +from fastcs_secop import SecopError, SecopQuirks from fastcs_secop.controllers import ( + SecopCommandController, SecopController, + SecopModuleController, ) @@ -38,3 +42,140 @@ async def test_ping_raises_disconnected_error_and_reconnect_fails(controller): controller.connect = AsyncMock(side_effect=ConnectionError) await controller.ping() controller.connect.assert_awaited() + + +async def test_ping_raises_disconnected_error_and_reconnect_works(controller): + with patch.object( + controller._connection, "send_query", AsyncMock(side_effect=[ConnectionError, "I'm alive"]) + ): + controller.connect = AsyncMock() + await controller.ping() + controller.connect.assert_awaited() + + +async def test_check_idn(): + controller = SecopController(settings=IPConnectionSettings("127.0.0.1", 0)) + controller._connection = AsyncMock() + + controller._connection.send_query.return_value = "ISSE&SINE2020,SECoP,foo,bar" + await controller.check_idn() + + controller._connection.send_query.return_value = "ISSE,SECoP,foo,bar" + await controller.check_idn() + + controller._connection.send_query.return_value = "blah,blah,blah,blah" + with pytest.raises(SecopError): + await controller.check_idn() + + controller._connection.send_query.return_value = "ISSE,not_secop,blah,blah" + with pytest.raises(SecopError): + await controller.check_idn() + + controller._connection.send_query.return_value = "a_random_device" + with pytest.raises(SecopError): + await controller.check_idn() + + +async def test_create_modules(): + controller = SecopController( + settings=IPConnectionSettings("127.0.0.1", 0), + quirks=SecopQuirks(skip_modules="a_skipped_module"), + ) + controller._connection = AsyncMock() + controller._connection.send_query.return_value = ( + "describing . " + + orjson.dumps( + { + "description": "some description", + "equipment_id": "some equipment id", + "modules": { + "a_cool_module": {"accessibles": {}}, + "another_cool_module": {"accessibles": {}}, + "a_skipped_module": {"accessibles": {}}, + }, + } + ).decode() + + "\n" + ) + + await controller.create_modules() + assert "a_cool_module" in controller.sub_controllers + assert "another_cool_module" in controller.sub_controllers + assert "a_skipped_module" not in controller.sub_controllers + + +async def test_create_modules_bad_description(): + controller = SecopController( + settings=IPConnectionSettings("127.0.0.1", 0), + quirks=SecopQuirks(skip_modules="a_skipped_module"), + ) + controller._connection = AsyncMock() + controller._connection.send_query.return_value = "a huge pile of nonsense\n" + + with pytest.raises(SecopError): + await controller.create_modules() + + +async def test_secop_module_controller_initialise(): + connection = AsyncMock() + controller = SecopModuleController( + connection=connection, + module_name="some_module", + module={ + "accessibles": { + "normal_accessible": {"datainfo": {"type": "int"}}, + "skipped_accessible": {"datainfo": {"type": "int"}}, + "raw_accessible": {"datainfo": {"type": "int"}}, + } + }, + quirks=SecopQuirks( + skip_accessibles=[("some_module", "skipped_accessible")], + raw_accessibles=[("some_module", "raw_accessible")], + ), + ) + + await controller.initialise() + + assert controller.attributes["normal_accessible"].dtype is int + assert controller.attributes["raw_accessible"].dtype is str + assert "skipped_accessible" not in controller.attributes + + +async def test_command_controller_execute_fails(): + connection = AsyncMock() + controller = SecopCommandController( + connection=connection, command_name="some_command", module_name="some_module", datainfo={} + ) + await controller.initialise() + + connection.send_query.return_value = "blah blah blah this isn't a valid response\n" + await controller.execute() # No exception thrown + + +async def test_command_controller_execute_no_args_no_return(): + connection = AsyncMock() + controller = SecopCommandController( + connection=connection, command_name="some_command", module_name="some_module", datainfo={} + ) + await controller.initialise() + + connection.send_query.return_value = "done some_module:some_command\n" + await controller.execute() + + +async def test_command_controller_execute(): + connection = AsyncMock() + controller = SecopCommandController( + connection=connection, + command_name="some_command", + module_name="some_module", + datainfo={"argument": {"type": "int"}, "result": {"type": "int"}}, + ) + await controller.initialise() + await controller.args.update(13) + + connection.send_query.return_value = 'done some_module:some_command [42, {"t": 123}]\n' + await controller.execute() + + connection.send_query.assert_awaited_once_with("do some_module:some_command 13\n") + assert controller.result.get() == 42 diff --git a/tests/test_io.py b/tests/test_io.py new file mode 100644 index 0000000..e647762 --- /dev/null +++ b/tests/test_io.py @@ -0,0 +1,243 @@ +import enum +from unittest.mock import AsyncMock, patch + +import numpy as np +import pytest +from fastcs.attributes import AttrRW +from fastcs.connections import IPConnection + +from fastcs_secop import SecopError +from fastcs_secop._util import secop_datainfo_to_fastcs_dtype +from fastcs_secop.io import ( + SecopAttributeIO, + SecopRawAttributeIO, + decode, + encode, + secop_change, + secop_read, +) + + +async def test_read_accessible_success(): + mock_connection = AsyncMock() + mock_connection.send_query.return_value = "reply some_module:some_accessible {'blah': 'blah'}\n" + + await secop_read( + connection=mock_connection, module_name="some_module", accessible_name="some_accessible" + ) + + mock_connection.send_query.assert_awaited_once_with("read some_module:some_accessible\n") + + +async def test_read_accessible_failure(): + mock_connection = AsyncMock() + mock_connection.send_query.return_value = ( + "error_read some_module:some_accessible {'blah': 'blah'}\n" + ) + + with pytest.raises(SecopError): + await secop_read( + connection=mock_connection, module_name="some_module", accessible_name="some_accessible" + ) + + mock_connection.send_query.assert_awaited_once_with("read some_module:some_accessible\n") + + +async def test_change_accessible_success(): + mock_connection = AsyncMock() + mock_connection.send_query.return_value = ( + "changed some_module:some_accessible {'blah': 'blah'}\n" + ) + + await secop_change( + connection=mock_connection, + module_name="some_module", + accessible_name="some_accessible", + encoded_value="{'blah': 'blah'}", + ) + + mock_connection.send_query.assert_awaited_once_with( + "change some_module:some_accessible {'blah': 'blah'}\n" + ) + + +async def test_change_accessible_failure(): + mock_connection = AsyncMock() + mock_connection.send_query.return_value = ( + "error_change some_module:some_accessible {'blah': 'blah'}\n" + ) + + with pytest.raises(SecopError): + await secop_change( + connection=mock_connection, + module_name="some_module", + accessible_name="some_accessible", + encoded_value="{'blah': 'blah'}", + ) + + mock_connection.send_query.assert_awaited_once_with( + "change some_module:some_accessible {'blah': 'blah'}\n" + ) + + +class DummyEnum(enum.Enum): + ONE = 1 + TWO = 2 + THREE = 3 + + +@pytest.mark.parametrize( + ("decoded", "datainfo", "encoded"), + [ + (1.23, {"type": "double"}, "1.23"), + (125.5, {"type": "scaled", "scale": 0.1}, "1255"), + (42, {"type": "int"}, "42"), + (True, {"type": "bool"}, "true"), + (DummyEnum.TWO, {"type": "enum", "members": {"ONE": 1, "TWO": 2, "THREE": 3}}, "2"), + ("hello", {"type": "string"}, '"hello"'), + (np.frombuffer(b"\0", dtype=np.uint8), {"type": "blob", "maxbytes": 512}, '"AA=="'), + (np.frombuffer(b"SECoP", dtype=np.uint8), {"type": "blob", "maxbytes": 512}, '"U0VDb1A="'), + ( + np.array([3, 4, 7, 2, 1], dtype=np.int32), + {"type": "array", "members": {"type": "int"}, "maxlen": 512}, + "[3,4,7,2,1]", + ), + ( + np.array([(300, "accelerating")], dtype=[("e0", np.int32), ("e1", " Date: Fri, 2 Jan 2026 12:25:18 +0000 Subject: [PATCH 15/19] Support matrix type --- doc/transports/epics_ca.md | 1 + examples/epics_ca.py | 1 + examples/epics_pva.py | 2 +- ruff.toml | 2 +- src/fastcs_secop/_util.py | 6 ++++++ src/fastcs_secop/controllers.py | 1 + src/fastcs_secop/io.py | 10 ++++++++++ tests/emulators/simple_secop/device.py | 6 ++++++ tests/test_io.py | 10 ++++++++++ tests/test_util.py | 4 ++++ 10 files changed, 41 insertions(+), 2 deletions(-) diff --git a/doc/transports/epics_ca.md b/doc/transports/epics_ca.md index d508a0b..750db9f 100644 --- a/doc/transports/epics_ca.md +++ b/doc/transports/epics_ca.md @@ -15,6 +15,7 @@ EPICS CA transport supports the following {external+secop:doc}`SECoP data types - string (`lsi`/`lso`) - blob (`waveform[char]`) - array of double/int/bool/{ref}`enum* ` (`waveform`) +- matrix, if the 'matrix' is 1-dimensional (`waveform`) - command (if arguments and return values are empty or one of the above types) Other data types can only be read or written in 'raw' mode. diff --git a/examples/epics_ca.py b/examples/epics_ca.py index 4f95a66..6954844 100644 --- a/examples/epics_ca.py +++ b/examples/epics_ca.py @@ -26,6 +26,7 @@ quirks = SecopQuirks( raw_tuple=True, raw_struct=True, + raw_matrix=True, max_description_length=40, raw_accessibles=[ ("valve_controller", "_domains_to_extract"), diff --git a/examples/epics_pva.py b/examples/epics_pva.py index e7e4702..afc69bf 100644 --- a/examples/epics_pva.py +++ b/examples/epics_pva.py @@ -34,7 +34,7 @@ DOCKER_GASFLOW = 10801 controller = SecopController( - settings=IPConnectionSettings(ip="127.0.0.1", port=DOCKER_GASFLOW), + settings=IPConnectionSettings(ip="127.0.0.1", port=LEWIS), quirks=quirks, ) diff --git a/ruff.toml b/ruff.toml index 112a5c6..4b4ab85 100644 --- a/ruff.toml +++ b/ruff.toml @@ -51,5 +51,5 @@ ignore = [ [lint.pylint] max-args = 8 -max-returns = 10 +max-returns = 15 max-branches = 15 diff --git a/src/fastcs_secop/_util.py b/src/fastcs_secop/_util.py index f2fbe81..4724794 100644 --- a/src/fastcs_secop/_util.py +++ b/src/fastcs_secop/_util.py @@ -33,6 +33,10 @@ class SecopQuirks: """If the accessible has an array type, read it in raw mode. """ + raw_matrix: bool = False + """If the accessible has a matrix type, read it in raw mode. + """ + raw_tuple: bool = False """If the accessible has a tuple type, read it in raw mode. @@ -156,5 +160,7 @@ def secop_datainfo_to_fastcs_dtype(datainfo: dict[str, Any], raw: bool = False) case "struct": structured_dtype = struct_structured_dtype(datainfo) return Table(structured_dtype) + case "matrix": + return Waveform(datainfo["elementtype"], shape=datainfo["maxlen"][::-1]) case _: raise SecopError(f"Invalid SECoP dtype for FastCS attribute: {datainfo['type']}") diff --git a/src/fastcs_secop/controllers.py b/src/fastcs_secop/controllers.py index 8c86d16..74cd967 100644 --- a/src/fastcs_secop/controllers.py +++ b/src/fastcs_secop/controllers.py @@ -149,6 +149,7 @@ def _is_raw(self, parameter_name: str, datainfo: dict[str, typing.Any]) -> bool: or (datainfo["type"] == "array" and self._quirks.raw_array) or (datainfo["type"] == "tuple" and self._quirks.raw_tuple) or (datainfo["type"] == "struct" and self._quirks.raw_struct) + or (datainfo["type"] == "matrix" and self._quirks.raw_matrix) ) async def initialise(self) -> None: diff --git a/src/fastcs_secop/io.py b/src/fastcs_secop/io.py index ecb39ae..52db207 100644 --- a/src/fastcs_secop/io.py +++ b/src/fastcs_secop/io.py @@ -127,6 +127,11 @@ def decode(raw_value: str, datainfo: dict[str, Any], attr: AttrR[T]) -> T: # no for k, v in cast(dict[str, Any], value).items(): arr[0][k] = v return arr + case "matrix": + lengths = value["len"][::-1] + return np.frombuffer( + base64.b64decode(value["blob"]), dtype=datainfo["elementtype"] + ).reshape(lengths) case _: return value @@ -164,6 +169,11 @@ def encode(value: T, datainfo: dict[str, Any]) -> str: for name in value.dtype.names: ans[name] = value[name][0] return orjson.dumps(ans, option=orjson.OPT_SERIALIZE_NUMPY).decode() + case "matrix": + assert isinstance(value, np.ndarray) + return orjson.dumps( + {"len": value.shape[::-1], "blob": base64.b64encode(value.tobytes()).decode()} + ).decode() case _: raise SecopError(f"Cannot handle SECoP dtype '{datainfo['type']}'") diff --git a/tests/emulators/simple_secop/device.py b/tests/emulators/simple_secop/device.py index 026400e..a5ed19e 100644 --- a/tests/emulators/simple_secop/device.py +++ b/tests/emulators/simple_secop/device.py @@ -164,6 +164,12 @@ def __init__(self): } }, ), + "matrix": Parameter( + {"len": [2, 3], "blob": "AACAPwAAAEAAAEBAAACAQAAAoEAAAMBA"}, + desc="a matrix parameter", + dtype="matrix", + extra_datainfo={"elementtype": " Date: Fri, 2 Jan 2026 17:22:17 +0000 Subject: [PATCH 16/19] Make most things private --- doc/contributing.md | 41 ++++++++++ doc/index.md | 1 + doc/spelling_wordlist.txt | 1 + doc/transports/epics_ca.md | 8 +- doc/transports/tango.md | 7 ++ examples/epics_ca.py | 14 ++-- examples/epics_pva.py | 14 ++-- src/fastcs_secop/__init__.py | 9 ++- .../{controllers.py => _controllers.py} | 80 +++++++++++-------- src/fastcs_secop/{io.py => _io.py} | 0 src/fastcs_secop/_util.py | 15 +++- tests/test_against_emulator.py | 2 +- tests/test_controller.py | 41 ++++++++-- tests/test_io.py | 18 ++--- 14 files changed, 184 insertions(+), 67 deletions(-) create mode 100644 doc/contributing.md rename src/fastcs_secop/{controllers.py => _controllers.py} (81%) rename src/fastcs_secop/{io.py => _io.py} (100%) diff --git a/doc/contributing.md b/doc/contributing.md new file mode 100644 index 0000000..f64ff8b --- /dev/null +++ b/doc/contributing.md @@ -0,0 +1,41 @@ +# Contributing + +The repository for this project is +[https://github.com/ISISComputingGroup/fastcs-secop](https://github.com/ISISComputingGroup/fastcs-secop). + +Contributions via GitHub issues and pull requests are welcome. If an issue or PR appears to have been ignored, it may have +simply been missed - email [ISISExperimentControls@stfc.ac.uk](mailto:ISISExperimentControls@stfc.ac.uk) if an issue +or PR appears to have been ignored accidentally. + +Some changes may require preparatory changes in [FastCS itself](https://github.com/DiamondLightSource/fastcs). + +## Developer installation + +To install a developer version of the library, run `pip install -e .[dev]` in a python virtual environment. +You can also use `uv pip install -e .[dev]` if you have the [`uv`](https://docs.astral.sh/uv/) tool installed. + +## Linting + +Linting is performed by `ruff` (formatting & linting) and `pyright` (type-checking). + +```shell +ruff format +ruff check --fix +pyright +``` + +## Documentation + +Documentation is built using `sphinx`. To get a local development build of the docs, use +`sphinx-autobuild doc _build --watch src`. + +Spell checking is run automatically in CI - if a word is correctly spelt but the spellchecker flags it, +add the word to `doc/spelling_wordlist.txt`. + +The spellchecker can be run manually using `sphinx-build -E -a -W --keep-going -b spelling doc _build` - this is best +run on a Windows machine due to differences in the system spelling dictionary between operating systems. + +## Tests + +Tests run via `pytest`. Some tests spawn a very basic lewis emulator on port 57677 to test a full communication +scenario. This is handled automatically by pytest, but may fail if port 57677 is already in use. diff --git a/doc/index.md b/doc/index.md index fcb45e8..e87fc06 100644 --- a/doc/index.md +++ b/doc/index.md @@ -23,6 +23,7 @@ transports/* :titlesonly: :caption: Reference +contributing limitations _api ``` diff --git a/doc/spelling_wordlist.txt b/doc/spelling_wordlist.txt index 43996ac..1a83487 100644 --- a/doc/spelling_wordlist.txt +++ b/doc/spelling_wordlist.txt @@ -9,6 +9,7 @@ deserialised deserialising enum enums +pytest SECoP struct structs diff --git a/doc/transports/epics_ca.md b/doc/transports/epics_ca.md index 750db9f..4afe083 100644 --- a/doc/transports/epics_ca.md +++ b/doc/transports/epics_ca.md @@ -2,7 +2,7 @@ EPICS CA transport requires `fastcs[epicsca]` to be installed. -EPICS CA has a maximum length of 40 on parameter descriptions. Set {py:obj}`fastcs_secop.SecopQuirks.max_description_length` to 40 to truncate descriptions. +EPICS CA has a maximum length of 40 on parameter descriptions. Set {py:obj}`SecopQuirks.max_description_length ` to 40 to truncate descriptions. ## Supported SECoP data types @@ -18,7 +18,11 @@ EPICS CA transport supports the following {external+secop:doc}`SECoP data types - matrix, if the 'matrix' is 1-dimensional (`waveform`) - command (if arguments and return values are empty or one of the above types) -Other data types can only be read or written in 'raw' mode. +Other data types can only be read or written in 'raw' mode. In particular, structs and tuples will need to be read in +raw mode; set {py:obj}`SecopQuirks.raw_struct ` and +{py:obj}`SecopQuirks.raw_tuple `. +{py:obj}`SecopQuirks.raw_matrix ` is also recommended, as only 1-D matrices are +supported by CA. ## Example CA IOC diff --git a/doc/transports/tango.md b/doc/transports/tango.md index 52777a5..cb91c1b 100644 --- a/doc/transports/tango.md +++ b/doc/transports/tango.md @@ -1,5 +1,12 @@ # Tango +:::{important} +While supported by FastCS, and therefore by {py:obj}`fastcs_secop`, Tango support has not been extensively tested. This +page documents some _known_ limitations. + +Modifications and improvements for Tango support are welcome. See {doc}`/contributing`. +::: + Tango transport requires `fastcs[tango]` to be installed. ## Supported SECoP data types diff --git a/examples/epics_ca.py b/examples/epics_ca.py index 6954844..e0b9078 100644 --- a/examples/epics_ca.py +++ b/examples/epics_ca.py @@ -1,5 +1,6 @@ """Example CA IOC using :py:obj:`fastcs_secop`.""" +import argparse import asyncio import logging import socket @@ -8,13 +9,17 @@ from fastcs.launch import FastCS from fastcs.logging import LogLevel, configure_logging -from fastcs_secop import SecopQuirks -from fastcs_secop.controllers import SecopController +from fastcs_secop import SecopController, SecopQuirks if __name__ == "__main__": from fastcs.transports import EpicsIOCOptions from fastcs.transports.epics.ca import EpicsCATransport + parser = argparse.ArgumentParser(description="Demo PVA ioc") + parser.add_argument("-i", "--ip", type=str, default="127.0.0.1", help="IP to connect to") + parser.add_argument("-p", "--port", type=int, help="Port to connect to") + args = parser.parse_args() + configure_logging(level=LogLevel.DEBUG) logging.basicConfig(level=LogLevel.DEBUG) @@ -34,11 +39,8 @@ ], ) - LEWIS = 57677 - DOCKER_GASFLOW = 10801 - controller = SecopController( - settings=IPConnectionSettings(ip="127.0.0.1", port=LEWIS), + settings=IPConnectionSettings(ip=args.ip, port=args.port), quirks=quirks, ) diff --git a/examples/epics_pva.py b/examples/epics_pva.py index afc69bf..7342b46 100644 --- a/examples/epics_pva.py +++ b/examples/epics_pva.py @@ -1,5 +1,6 @@ """Example PVA IOC using :py:obj:`fastcs_secop`.""" +import argparse import asyncio import logging import socket @@ -8,13 +9,17 @@ from fastcs.launch import FastCS from fastcs.logging import LogLevel, configure_logging -from fastcs_secop import SecopQuirks -from fastcs_secop.controllers import SecopController +from fastcs_secop import SecopController, SecopQuirks if __name__ == "__main__": from fastcs.transports import EpicsIOCOptions from fastcs.transports.epics.pva import EpicsPVATransport + parser = argparse.ArgumentParser(description="Demo PVA ioc") + parser.add_argument("-i", "--ip", type=str, default="127.0.0.1", help="IP to connect to") + parser.add_argument("-p", "--port", type=int, help="Port to connect to") + args = parser.parse_args() + configure_logging(level=LogLevel.DEBUG) logging.basicConfig(level=LogLevel.DEBUG) @@ -30,11 +35,8 @@ ], ) - LEWIS = 57677 - DOCKER_GASFLOW = 10801 - controller = SecopController( - settings=IPConnectionSettings(ip="127.0.0.1", port=LEWIS), + settings=IPConnectionSettings(ip=args.ip, port=args.port), quirks=quirks, ) diff --git a/src/fastcs_secop/__init__.py b/src/fastcs_secop/__init__.py index 403f068..ab1725a 100644 --- a/src/fastcs_secop/__init__.py +++ b/src/fastcs_secop/__init__.py @@ -1,5 +1,12 @@ """SECoP support using FastCS.""" +from fastcs_secop._controllers import SecopCommandController, SecopController, SecopModuleController from fastcs_secop._util import SecopError, SecopQuirks -__all__ = ["SecopError", "SecopQuirks"] +__all__ = [ + "SecopCommandController", + "SecopController", + "SecopError", + "SecopModuleController", + "SecopQuirks", +] diff --git a/src/fastcs_secop/controllers.py b/src/fastcs_secop/_controllers.py similarity index 81% rename from src/fastcs_secop/controllers.py rename to src/fastcs_secop/_controllers.py index 74cd967..b3f6fd0 100644 --- a/src/fastcs_secop/controllers.py +++ b/src/fastcs_secop/_controllers.py @@ -10,12 +10,7 @@ from fastcs.controllers import Controller from fastcs.methods import command, scan -from fastcs_secop import SecopQuirks -from fastcs_secop._util import ( - SecopError, - secop_datainfo_to_fastcs_dtype, -) -from fastcs_secop.io import ( +from fastcs_secop._io import ( SecopAttributeIO, SecopAttributeIORef, SecopRawAttributeIO, @@ -23,6 +18,7 @@ decode, encode, ) +from fastcs_secop._util import SecopError, SecopQuirks, is_raw, secop_datainfo_to_fastcs_dtype logger = getLogger(__name__) @@ -37,6 +33,7 @@ def __init__( module_name: str, command_name: str, datainfo: dict[str, typing.Any], + quirks: SecopQuirks, ) -> None: """Subcontroller for a SECoP command. @@ -45,27 +42,40 @@ def __init__( module_name: The module in which this command is defined. command_name: The name of the command. datainfo: The datainfo dictionary for this command. + quirks: The quirks configuration (see :py:obj:`~fastcs_secop.SecopQuirks`). """ super().__init__() - self.connection = connection - self.module_name = module_name - self.command_name = command_name - self.datainfo = datainfo + self._connection = connection + self._module_name = module_name + self._command_name = command_name + self._datainfo = datainfo + self._quirks = quirks + + self.raw_args = self._datainfo.get("argument") is not None and is_raw( + self._module_name, self._command_name, self._datainfo["argument"], self._quirks + ) + self.raw_result = self._datainfo.get("result") is not None and is_raw( + self._module_name, self._command_name, self._datainfo["result"], self._quirks + ) async def initialise(self) -> None: """Initialise the command controller. This will set up PVs for ``Args`` and ``Result`` (if they have a type). """ - if self.datainfo.get("argument") is not None: - args_type = secop_datainfo_to_fastcs_dtype(self.datainfo["argument"]) + if self._datainfo.get("argument") is not None: + args_type = secop_datainfo_to_fastcs_dtype( + self._datainfo["argument"], raw=self.raw_args + ) else: args_type = None - if self.datainfo.get("result") is not None: - result_type = secop_datainfo_to_fastcs_dtype(self.datainfo["result"]) + if self._datainfo.get("result") is not None: + result_type = secop_datainfo_to_fastcs_dtype( + self._datainfo["result"], raw=self.raw_result + ) else: result_type = None @@ -82,16 +92,19 @@ async def initialise(self) -> None: @command() async def execute(self) -> None: """Execute the command.""" - prefix = f"do {self.module_name}:{self.command_name}" - response_prefix = f"done {self.module_name}:{self.command_name}" + prefix = f"do {self._module_name}:{self._command_name}" + response_prefix = f"done {self._module_name}:{self._command_name}" if self.args is not None: - cmd = f"{prefix} {encode(self.args.get(), self.datainfo['argument'])}\n" + if self.raw_args: + cmd = f"{prefix} {self.args.get()}\n" + else: + cmd = f"{prefix} {encode(self.args.get(), self._datainfo['argument'])}\n" else: cmd = f"{prefix}\n" logger.debug("Sending command: '%s'", cmd) - response = await self.connection.send_query(cmd) + response = await self._connection.send_query(cmd) logger.debug("Response: '%s'", response) response = response.strip() @@ -102,7 +115,10 @@ async def execute(self) -> None: response = response[len(response_prefix) :].strip() if self.result is not None: - await self.result.update(decode(response, self.datainfo["result"], self.result)) + if self.raw_result: + await self.result.update(orjson.dumps(orjson.loads(response)[0]).decode()) + else: + await self.result.update(decode(response, self._datainfo["result"], self.result)) class SecopModuleController(Controller): @@ -143,15 +159,6 @@ def __init__( ] ) - def _is_raw(self, parameter_name: str, datainfo: dict[str, typing.Any]) -> bool: - return ( - ((self._module_name, parameter_name) in self._quirks.raw_accessibles) - or (datainfo["type"] == "array" and self._quirks.raw_array) - or (datainfo["type"] == "tuple" and self._quirks.raw_tuple) - or (datainfo["type"] == "struct" and self._quirks.raw_struct) - or (datainfo["type"] == "matrix" and self._quirks.raw_matrix) - ) - async def initialise(self) -> None: """Create attributes for all accessibles in this SECoP module.""" for parameter_name, parameter in self._module["accessibles"].items(): @@ -165,7 +172,7 @@ async def initialise(self) -> None: attr_cls = AttrR if parameter.get("readonly", False) else AttrRW - raw = self._is_raw(parameter_name, datainfo) + raw = is_raw(self._module_name, parameter_name, datainfo, self._quirks) if raw: io_ref = SecopRawAttributeIORef( @@ -187,6 +194,7 @@ async def initialise(self) -> None: command_name=parameter_name, connection=self._connection, datainfo=datainfo, + quirks=self._quirks, ) self.add_sub_controller(parameter_name, command_controller) await command_controller.initialise() @@ -217,7 +225,7 @@ def __init__(self, settings: IPConnectionSettings, quirks: SecopQuirks | None = """ self._ip_settings = settings self._connection = IPConnection() - self.quirks = quirks or SecopQuirks() + self._quirks = quirks or SecopQuirks() super().__init__() @@ -309,9 +317,9 @@ async def initialise(self) -> None: await self.connect() await self.check_idn() await self.deactivate() - await self.create_modules() + await self._create_modules() - async def create_modules(self) -> None: + async def _create_modules(self) -> None: """Create subcontrollers for each SECoP module.""" descriptor = await self._connection.send_query("describe\n") if not descriptor.startswith("describing . "): @@ -323,19 +331,21 @@ async def create_modules(self) -> None: equipment_id = descriptor["equipment_id"] logger.info("SECoP equipment_id = '%s', description = '%s'", equipment_id, description) - logger.debug("descriptor = %s", orjson.dumps(descriptor, option=orjson.OPT_INDENT_2)) + logger.debug( + "descriptor = %s", orjson.dumps(descriptor, option=orjson.OPT_INDENT_2).decode() + ) modules = descriptor["modules"] for module_name, module in modules.items(): - if module_name in self.quirks.skip_modules: + if module_name in self._quirks.skip_modules: continue logger.debug("Creating subcontroller for module %s", module_name) module_controller = SecopModuleController( connection=self._connection, module_name=module_name, module=module, - quirks=self.quirks, + quirks=self._quirks, ) await module_controller.initialise() self.add_sub_controller(name=module_name, sub_controller=module_controller) diff --git a/src/fastcs_secop/io.py b/src/fastcs_secop/_io.py similarity index 100% rename from src/fastcs_secop/io.py rename to src/fastcs_secop/_io.py diff --git a/src/fastcs_secop/_util.py b/src/fastcs_secop/_util.py index 4724794..eb19400 100644 --- a/src/fastcs_secop/_util.py +++ b/src/fastcs_secop/_util.py @@ -1,4 +1,5 @@ import enum +import typing from collections.abc import Collection from dataclasses import dataclass, field from typing import Any @@ -115,7 +116,7 @@ def secop_datainfo_to_fastcs_dtype(datainfo: dict[str, Any], raw: bool = False) """ if raw: - return String() + return String(2048) min_val = datainfo.get("min") max_val = datainfo.get("max") @@ -164,3 +165,15 @@ def secop_datainfo_to_fastcs_dtype(datainfo: dict[str, Any], raw: bool = False) return Waveform(datainfo["elementtype"], shape=datainfo["maxlen"][::-1]) case _: raise SecopError(f"Invalid SECoP dtype for FastCS attribute: {datainfo['type']}") + + +def is_raw( + module_name: str, parameter_name: str, datainfo: dict[str, typing.Any], quirks: SecopQuirks +) -> bool: + return ( + ((module_name, parameter_name) in quirks.raw_accessibles) + or (datainfo["type"] == "array" and quirks.raw_array) + or (datainfo["type"] == "tuple" and quirks.raw_tuple) + or (datainfo["type"] == "struct" and quirks.raw_struct) + or (datainfo["type"] == "matrix" and quirks.raw_matrix) + ) diff --git a/tests/test_against_emulator.py b/tests/test_against_emulator.py index 96c7a60..fb90bad 100644 --- a/tests/test_against_emulator.py +++ b/tests/test_against_emulator.py @@ -12,7 +12,7 @@ from fastcs.connections import IPConnectionSettings from fastcs.logging import LogLevel, configure_logging -from fastcs_secop.controllers import SecopController +from fastcs_secop import SecopController configure_logging(level=LogLevel.TRACE) diff --git a/tests/test_controller.py b/tests/test_controller.py index 70a0306..52fd95c 100644 --- a/tests/test_controller.py +++ b/tests/test_controller.py @@ -4,11 +4,12 @@ import pytest from fastcs.connections import IPConnectionSettings -from fastcs_secop import SecopError, SecopQuirks -from fastcs_secop.controllers import ( +from fastcs_secop import ( SecopCommandController, SecopController, + SecopError, SecopModuleController, + SecopQuirks, ) @@ -98,7 +99,7 @@ async def test_create_modules(): + "\n" ) - await controller.create_modules() + await controller._create_modules() assert "a_cool_module" in controller.sub_controllers assert "another_cool_module" in controller.sub_controllers assert "a_skipped_module" not in controller.sub_controllers @@ -113,7 +114,7 @@ async def test_create_modules_bad_description(): controller._connection.send_query.return_value = "a huge pile of nonsense\n" with pytest.raises(SecopError): - await controller.create_modules() + await controller._create_modules() async def test_secop_module_controller_initialise(): @@ -144,7 +145,11 @@ async def test_secop_module_controller_initialise(): async def test_command_controller_execute_fails(): connection = AsyncMock() controller = SecopCommandController( - connection=connection, command_name="some_command", module_name="some_module", datainfo={} + connection=connection, + command_name="some_command", + module_name="some_module", + datainfo={}, + quirks=SecopQuirks(), ) await controller.initialise() @@ -155,7 +160,11 @@ async def test_command_controller_execute_fails(): async def test_command_controller_execute_no_args_no_return(): connection = AsyncMock() controller = SecopCommandController( - connection=connection, command_name="some_command", module_name="some_module", datainfo={} + connection=connection, + command_name="some_command", + module_name="some_module", + datainfo={}, + quirks=SecopQuirks(), ) await controller.initialise() @@ -170,6 +179,7 @@ async def test_command_controller_execute(): command_name="some_command", module_name="some_module", datainfo={"argument": {"type": "int"}, "result": {"type": "int"}}, + quirks=SecopQuirks(), ) await controller.initialise() await controller.args.update(13) @@ -179,3 +189,22 @@ async def test_command_controller_execute(): connection.send_query.assert_awaited_once_with("do some_module:some_command 13\n") assert controller.result.get() == 42 + + +async def test_command_controller_execute_raw_args_and_result(): + connection = AsyncMock() + controller = SecopCommandController( + connection=connection, + command_name="some_command", + module_name="some_module", + datainfo={"argument": {"type": "tuple"}, "result": {"type": "tuple"}}, + quirks=SecopQuirks(raw_tuple=True), + ) + await controller.initialise() + await controller.args.update("[13]") + + connection.send_query.return_value = 'done some_module:some_command [[42], {"t": 123}]\n' + await controller.execute() + + connection.send_query.assert_awaited_once_with("do some_module:some_command [13]\n") + assert controller.result.get() == "[42]" diff --git a/tests/test_io.py b/tests/test_io.py index 02424cb..7482616 100644 --- a/tests/test_io.py +++ b/tests/test_io.py @@ -7,8 +7,7 @@ from fastcs.connections import IPConnection from fastcs_secop import SecopError -from fastcs_secop._util import secop_datainfo_to_fastcs_dtype -from fastcs_secop.io import ( +from fastcs_secop._io import ( SecopAttributeIO, SecopRawAttributeIO, decode, @@ -16,6 +15,7 @@ secop_change, secop_read, ) +from fastcs_secop._util import secop_datainfo_to_fastcs_dtype async def test_read_accessible_success(): @@ -185,8 +185,8 @@ async def test_attribute_io_update(io_cls, expected): io = io_cls(connection=connection) with ( - patch("fastcs_secop.io.secop_read", return_value='[123.456, {"t": 5, "e": 7}]'), - patch("fastcs_secop.io.decode", return_value=123.456), + patch("fastcs_secop._io.secop_read", return_value='[123.456, {"t": 5, "e": 7}]'), + patch("fastcs_secop._io.decode", return_value=123.456), ): await io.update(attr) @@ -207,7 +207,7 @@ async def test_attribute_io_update_fails(io_cls, error): attr = AsyncMock(spec=AttrRW) io = io_cls(connection=connection) - with patch("fastcs_secop.io.secop_read", side_effect=error): + with patch("fastcs_secop._io.secop_read", side_effect=error): await io.update(attr) @@ -224,8 +224,8 @@ async def test_attribute_io_send(io_cls, expected): io = io_cls(connection=connection) with ( - patch("fastcs_secop.io.secop_change") as mock_change, - patch("fastcs_secop.io.encode", return_value="123.456"), + patch("fastcs_secop._io.secop_change") as mock_change, + patch("fastcs_secop._io.encode", return_value="123.456"), ): await io.send(attr, 123.456) @@ -247,7 +247,7 @@ async def test_attribute_io_send_fails(io_cls, error): io = io_cls(connection=connection) with ( - patch("fastcs_secop.io.secop_change", side_effect=error), - patch("fastcs_secop.io.encode", return_value="123.456"), + patch("fastcs_secop._io.secop_change", side_effect=error), + patch("fastcs_secop._io.encode", return_value="123.456"), ): await io.send(attr, 123.456) From 7ce7b1606651df3d65134ac07f6d7d58e8805f67 Mon Sep 17 00:00:00 2001 From: Tom Willemsen Date: Sat, 3 Jan 2026 12:47:33 +0000 Subject: [PATCH 17/19] improve docs --- doc/_static/css/custom.css | 9 +++++++++ doc/conf.py | 3 +++ doc/contributing.md | 7 ++++--- doc/logo.svg | 18 ++++++++++++++++++ doc/transports/epics_ca.md | 1 + doc/transports/epics_pva.md | 1 + pyproject.toml | 1 + src/fastcs_secop/_controllers.py | 30 ++++++++++++++++++++++++++++-- src/fastcs_secop/_io.py | 6 ++++-- src/fastcs_secop/_util.py | 16 +++++++++++++++- 10 files changed, 84 insertions(+), 8 deletions(-) create mode 100644 doc/logo.svg diff --git a/doc/_static/css/custom.css b/doc/_static/css/custom.css index 1a1c10a..e491bfe 100644 --- a/doc/_static/css/custom.css +++ b/doc/_static/css/custom.css @@ -5,3 +5,12 @@ a:hover { .wy-menu-vertical p.caption { color: #d2b48c; } + +.sphinx-codeautolink-a{ + border-bottom-color: #d2b48c; + border-bottom-style: solid; + border-bottom-width: 1px; +} +.sphinx-codeautolink-a:hover{ + color: #888888; +} diff --git a/doc/conf.py b/doc/conf.py index 73821b4..dcbde61 100644 --- a/doc/conf.py +++ b/doc/conf.py @@ -45,6 +45,8 @@ "sphinx.ext.viewcode", # Mermaid diagrams "sphinxcontrib.mermaid", + # Documentation links in code blocks + "sphinx_codeautolink", ] mermaid_d3_zoom = True napoleon_google_docstring = True @@ -73,6 +75,7 @@ html_css_files = [ "css/custom.css", ] +html_logo = "logo.svg" autoclass_content = "init" myst_heading_anchors = 7 diff --git a/doc/contributing.md b/doc/contributing.md index f64ff8b..9815608 100644 --- a/doc/contributing.md +++ b/doc/contributing.md @@ -16,7 +16,8 @@ You can also use `uv pip install -e .[dev]` if you have the [`uv`](https://docs. ## Linting -Linting is performed by `ruff` (formatting & linting) and `pyright` (type-checking). +Linting is performed by [ruff](https://docs.astral.sh/ruff/) (formatting & linting) and +[pyright](https://github.com/microsoft/pyright) (type-checking). ```shell ruff format @@ -26,8 +27,8 @@ pyright ## Documentation -Documentation is built using `sphinx`. To get a local development build of the docs, use -`sphinx-autobuild doc _build --watch src`. +Documentation is built using [sphinx](https://www.sphinx-doc.org/en/master/). +To get a local development build of the docs, use `sphinx-autobuild doc _build --watch src`. Spell checking is run automatically in CI - if a word is correctly spelt but the spellchecker flags it, add the word to `doc/spelling_wordlist.txt`. diff --git a/doc/logo.svg b/doc/logo.svg new file mode 100644 index 0000000..f79f3d5 --- /dev/null +++ b/doc/logo.svg @@ -0,0 +1,18 @@ + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/doc/transports/epics_ca.md b/doc/transports/epics_ca.md index 4afe083..bdef96f 100644 --- a/doc/transports/epics_ca.md +++ b/doc/transports/epics_ca.md @@ -24,6 +24,7 @@ raw mode; set {py:obj}`SecopQuirks.raw_struct ` is also recommended, as only 1-D matrices are supported by CA. +{#example_ca_ioc} ## Example CA IOC :::python diff --git a/doc/transports/epics_pva.md b/doc/transports/epics_pva.md index 83a1561..1eac932 100644 --- a/doc/transports/epics_pva.md +++ b/doc/transports/epics_pva.md @@ -27,6 +27,7 @@ Other data types can only be read or written in 'raw' mode. SECoP modules can be found under the top-level PVI structure, while SECoP accessibles can be found under `module_name:PVI`. This means that the IOC is self-describing to downstream clients. +{#example_pva_ioc} ## Example PVA IOC :::python diff --git a/pyproject.toml b/pyproject.toml index cd3671c..ed25b12 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -37,6 +37,7 @@ doc = [ "sphinx_rtd_theme", "myst_parser", "sphinx-autobuild", + "sphinx-codeautolink", "sphinxcontrib-mermaid", "sphinxcontrib-spelling", ] diff --git a/src/fastcs_secop/_controllers.py b/src/fastcs_secop/_controllers.py index b3f6fd0..f25854d 100644 --- a/src/fastcs_secop/_controllers.py +++ b/src/fastcs_secop/_controllers.py @@ -37,6 +37,9 @@ def __init__( ) -> None: """Subcontroller for a SECoP command. + This class is automatically added as a subcontroller by + :py:obj:`SecopModuleController` for command-type parameters. + Args: connection: The connection to use. module_name: The module in which this command is defined. @@ -134,8 +137,8 @@ def __init__( ) -> None: """FastCS controller for a SECoP module. - Instances of this class are added as subcontrollers by - :py:obj:`SecopController`. + This class is automatically added as a subcontroller by + :py:obj:`SecopController` for each present SECoP module. Args: connection: The connection to use. @@ -217,6 +220,29 @@ class SecopController(Controller): def __init__(self, settings: IPConnectionSettings, quirks: SecopQuirks | None = None) -> None: """FastCS Controller for a SECoP node. + The intended usage is via :py:obj:`fastcs.control_system.FastCS`: + + .. code-block:: python + + from fastcs_secop import SecopController, SecopQuirks + from fastcs.control_system import FastCS + + controller = SecopController( + settings=IPConnectionSettings(ip="127.0.0.1", port=1234), + quirks=SecopQuirks(...), + ) + + transports = [...] + + fastcs = FastCS( + controller, + transports, + ) + fastcs.run() + + See Also: + :ref:`example_ca_ioc` and :ref:`example_pva_ioc` for examples of full configurations + Args: settings: The communication settings (e.g. IP address, port) at which the SECoP node is reachable. diff --git a/src/fastcs_secop/_io.py b/src/fastcs_secop/_io.py index 52db207..3bcb9e3 100644 --- a/src/fastcs_secop/_io.py +++ b/src/fastcs_secop/_io.py @@ -212,6 +212,7 @@ async def send(self, attr: AttrW[T, SecopAttributeIORef], value: T) -> None: encoded_value, ) # Ugly, but I can't find a public alternative... + # https://github.com/DiamondLightSource/FastCS/pull/292 await attr._call_sync_setpoint_callbacks(value) # noqa: SLF001 except ConnectionError: # Reconnect will be attempted in a periodic scan task @@ -223,8 +224,8 @@ async def send(self, attr: AttrW[T, SecopAttributeIORef], value: T) -> None: class SecopRawAttributeIO(AttributeIO[str, SecopRawAttributeIORef]): """Raw IO for a SECoP parameter of any type other than 'command'. - For "raw" IO, no serialization/deserialization is performed. All values are transmitted - to/from FastCS as strings. It is up to the client to interpret those strings correctly. + For "raw" IO, all values are transmitted to/from FastCS as strings. + It is up to the client to interpret those strings correctly. This is intended as a fallback mode for transports which cannot represent complex data types. @@ -262,6 +263,7 @@ async def send(self, attr: AttrW[str, SecopRawAttributeIORef], value: str) -> No value, ) # Ugly, but I can't find a public alternative... + # https://github.com/DiamondLightSource/FastCS/pull/292 await attr._call_sync_setpoint_callbacks(value) # noqa: SLF001 except ConnectionError: # Reconnect will be attempted in a periodic scan task diff --git a/src/fastcs_secop/_util.py b/src/fastcs_secop/_util.py index eb19400..900facd 100644 --- a/src/fastcs_secop/_util.py +++ b/src/fastcs_secop/_util.py @@ -28,19 +28,31 @@ class SecopQuirks: """Skip creating any listed ``(module_name, accessible_name)`` tuples.""" raw_accessibles: Collection[tuple[str, str]] = field(default_factory=list) - """Create any listed ``(module_name, accessible_name)`` tuples in 'raw' mode.""" + """Create any listed ``(module_name, accessible_name)`` tuples in raw mode. + + JSON for the specified accessibles will be treated as strings. + """ raw_array: bool = False """If the accessible has an array type, read it in raw mode. + + JSON values for any array-type accessible will be treated as strings. """ raw_matrix: bool = False """If the accessible has a matrix type, read it in raw mode. + + JSON values for any matrix-type accessible will be treated as strings. + + This is useful for transports which cannot represent arbitrary N-dimensional + arrays. """ raw_tuple: bool = False """If the accessible has a tuple type, read it in raw mode. + JSON values for any tuple-type accessible will be treated as strings. + This is useful for transports which do not support the FastCS :py:obj:`~fastcs.datatypes.table.Table` type. """ @@ -48,6 +60,8 @@ class SecopQuirks: raw_struct: bool = False """If the accessible has a struct type, read it in raw mode. + JSON values for any struct-type accessible will be treated as strings. + This is useful for transports which do not support the FastCS :py:obj:`~fastcs.datatypes.table.Table` type. """ From 1a8796f3a79fecca2f646741725bc87c80d56d9a Mon Sep 17 00:00:00 2001 From: Tom Willemsen Date: Sun, 4 Jan 2026 09:57:07 +0000 Subject: [PATCH 18/19] AI review comments --- .github/workflows/release.yml | 2 +- pyproject.toml | 1 + src/fastcs_secop/_controllers.py | 65 +++++++++++++++++++------------- src/fastcs_secop/_io.py | 12 +++--- src/fastcs_secop/_util.py | 2 +- tests/test_against_emulator.py | 1 - tests/test_controller.py | 17 ++++++++- 7 files changed, 63 insertions(+), 37 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 74b53bf..241085f 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -16,7 +16,7 @@ jobs: - name: Set up Python uses: actions/setup-python@v6 with: - python-version: "3.13" + python-version: "3.12" - name: Install pypa/build run: >- python3 -m diff --git a/pyproject.toml b/pyproject.toml index ed25b12..601c37f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -27,6 +27,7 @@ classifiers = [ ] dependencies = [ + # Pinned until https://github.com/DiamondLightSource/FastCS/pull/295 is merged and released "fastcs @ git+https://github.com/Tom-Willemsen/FastCS@guard_methods_that_dont_work_on_windows", "orjson", ] diff --git a/src/fastcs_secop/_controllers.py b/src/fastcs_secop/_controllers.py index f25854d..27b39b8 100644 --- a/src/fastcs_secop/_controllers.py +++ b/src/fastcs_secop/_controllers.py @@ -95,33 +95,44 @@ async def initialise(self) -> None: @command() async def execute(self) -> None: """Execute the command.""" - prefix = f"do {self._module_name}:{self._command_name}" - response_prefix = f"done {self._module_name}:{self._command_name}" - - if self.args is not None: - if self.raw_args: - cmd = f"{prefix} {self.args.get()}\n" - else: - cmd = f"{prefix} {encode(self.args.get(), self._datainfo['argument'])}\n" - else: - cmd = f"{prefix}\n" - - logger.debug("Sending command: '%s'", cmd) - response = await self._connection.send_query(cmd) - logger.debug("Response: '%s'", response) - - response = response.strip() - if not response.startswith(response_prefix): - logger.warning("command '%s' failed (response='%s')", prefix, response) - return - - response = response[len(response_prefix) :].strip() - - if self.result is not None: - if self.raw_result: - await self.result.update(orjson.dumps(orjson.loads(response)[0]).decode()) + try: + prefix = f"do {self._module_name}:{self._command_name}" + response_prefix = f"done {self._module_name}:{self._command_name}" + + if self.args is not None: + if self.raw_args: + cmd = f"{prefix} {self.args.get()}\n" + else: + cmd = f"{prefix} {encode(self.args.get(), self._datainfo['argument'])}\n" else: - await self.result.update(decode(response, self._datainfo["result"], self.result)) + cmd = f"{prefix}\n" + + logger.debug("Sending command: '%s'", cmd) + response = await self._connection.send_query(cmd) + logger.debug("Response: '%s'", response) + + response = response.strip() + if not response.startswith(response_prefix): + logger.warning("command '%s' failed (response='%s')", prefix, response) + return + + response = response[len(response_prefix) :].strip() + + if self.result is not None: + if self.raw_result: + await self.result.update(orjson.dumps(orjson.loads(response)[0]).decode()) + else: + await self.result.update( + decode(response, self._datainfo["result"], self.result) + ) + except Exception as e: + logger.error( + "command %s:%s failed: %s: %s", + self._module_name, + self._command_name, + e.__class__.__name__, + e, + ) class SecopModuleController(Controller): @@ -318,7 +329,7 @@ async def check_idn(self) -> None: f"Not a SECoP device?" ) - print(f"Connected to SECoP device with IDN='{identification}'.") + logger.info("Connected to SECoP device with IDN='%s'.", identification) async def initialise(self) -> None: """Set up FastCS for this SECoP node. diff --git a/src/fastcs_secop/_io.py b/src/fastcs_secop/_io.py index 3bcb9e3..b397de5 100644 --- a/src/fastcs_secop/_io.py +++ b/src/fastcs_secop/_io.py @@ -151,12 +151,12 @@ def encode(value: T, datainfo: dict[str, Any]) -> str: assert isinstance(value, enum.Enum) return orjson.dumps(value.value).decode() case "scaled": - return orjson.dumps(round(value / datainfo["scale"])).decode() + val = round(value / datainfo["scale"]) + assert isinstance(val, int) + return orjson.dumps(val).decode() case "blob": assert isinstance(value, np.ndarray) - return orjson.dumps( - base64.b64encode("".join(chr(c) for c in value).encode("utf-8")).decode() - ).decode() + return orjson.dumps(base64.b64encode(value.tobytes()).decode()).decode() case "array": return orjson.dumps(value, option=orjson.OPT_SERIALIZE_NUMPY).decode() case "tuple": @@ -217,8 +217,8 @@ async def send(self, attr: AttrW[T, SecopAttributeIORef], value: T) -> None: except ConnectionError: # Reconnect will be attempted in a periodic scan task pass - except Exception: - logger.exception("Exception during send()") + except Exception as e: + logger.error("Exception during send() for %s: %s: %s", attr, e.__class__.__name__, e) class SecopRawAttributeIO(AttributeIO[str, SecopRawAttributeIORef]): diff --git a/src/fastcs_secop/_util.py b/src/fastcs_secop/_util.py index 900facd..2d409d4 100644 --- a/src/fastcs_secop/_util.py +++ b/src/fastcs_secop/_util.py @@ -159,7 +159,7 @@ def secop_datainfo_to_fastcs_dtype(datainfo: dict[str, Any], raw: bool = False) case "bool": return Bool() case "enum": - enum_type = enum.Enum("enum_type", datainfo["members"]) + enum_type = enum.Enum("GeneratedSecopEnum", datainfo["members"]) return Enum(enum_type) case "string": return String() diff --git a/tests/test_against_emulator.py b/tests/test_against_emulator.py index fb90bad..716a554 100644 --- a/tests/test_against_emulator.py +++ b/tests/test_against_emulator.py @@ -45,7 +45,6 @@ async def controller(): ip="127.0.0.1", port=57677, ), - quirks={}, ) for _ in range(100): diff --git a/tests/test_controller.py b/tests/test_controller.py index 52fd95c..8519354 100644 --- a/tests/test_controller.py +++ b/tests/test_controller.py @@ -142,7 +142,7 @@ async def test_secop_module_controller_initialise(): assert "skipped_accessible" not in controller.attributes -async def test_command_controller_execute_fails(): +async def test_command_controller_execute_invalid_response(): connection = AsyncMock() controller = SecopCommandController( connection=connection, @@ -157,6 +157,21 @@ async def test_command_controller_execute_fails(): await controller.execute() # No exception thrown +async def test_command_controller_execute_fails(): + connection = AsyncMock() + controller = SecopCommandController( + connection=connection, + command_name="some_command", + module_name="some_module", + datainfo={}, + quirks=SecopQuirks(), + ) + await controller.initialise() + + connection.send_query.side_effect = Exception + await controller.execute() # No exception thrown + + async def test_command_controller_execute_no_args_no_return(): connection = AsyncMock() controller = SecopCommandController( From ca58753f9eb6fa4c62b76b29b0a586d7981e142e Mon Sep 17 00:00:00 2001 From: Tom Willemsen Date: Sat, 10 Jan 2026 09:14:32 +0000 Subject: [PATCH 19/19] Ignore RUF067 in tests --- ruff.toml | 1 + src/fastcs_secop/_controllers.py | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/ruff.toml b/ruff.toml index 4b4ab85..8221db2 100644 --- a/ruff.toml +++ b/ruff.toml @@ -44,6 +44,7 @@ ignore = [ "PLR0914", # Allow complex tests "PLC2701", # Allow tests to import "private" things "SLF001", # Allow tests to use "private" things + "RUF067", # Standard pattern for LEWiS emulators ] "doc/conf.py" = [ "D100" diff --git a/src/fastcs_secop/_controllers.py b/src/fastcs_secop/_controllers.py index 27b39b8..4215376 100644 --- a/src/fastcs_secop/_controllers.py +++ b/src/fastcs_secop/_controllers.py @@ -113,7 +113,7 @@ async def execute(self) -> None: response = response.strip() if not response.startswith(response_prefix): - logger.warning("command '%s' failed (response='%s')", prefix, response) + logger.error("command '%s' failed (response='%s')", prefix, response) return response = response[len(response_prefix) :].strip()