From aa856cf33c9b8a29e2886f051263a4266e2a21df Mon Sep 17 00:00:00 2001 From: Dmitri Gavrilov Date: Sun, 4 May 2025 14:27:51 -0400 Subject: [PATCH 1/9] ENH: add parameters for setting 0MQ encoding --- bluesky_httpserver/app.py | 6 +++++- .../config_schemas/service_configuration.yml | 4 ++++ ...ff4e4fcc7_add_write_scopes_to_default_roles.py | 2 +- bluesky_httpserver/schemas.py | 3 +++ bluesky_httpserver/tests/conftest.py | 3 +++ bluesky_httpserver/tests/test_console_output.py | 15 ++++++++++++--- docs/source/configuration.rst | 4 ++++ 7 files changed, 32 insertions(+), 5 deletions(-) diff --git a/bluesky_httpserver/app.py b/bluesky_httpserver/app.py index a659784..4779f4c 100644 --- a/bluesky_httpserver/app.py +++ b/bluesky_httpserver/app.py @@ -307,9 +307,12 @@ async def purge_expired_sessions_and_api_keys(): "QSERVER_ZMQ_INFO_ADDRESS to pass address of 0MQ information socket to HTTP Server." ) + zmq_encoding = os.getenv("QSERVER_ZMQ_ENCODING", None) + # Check if ZMQ setting were specified in config file. Overrid the parameters from EVs. zmq_control_addr = server_settings["qserver_zmq_configuration"].get("control_address", zmq_control_addr) zmq_info_addr = server_settings["qserver_zmq_configuration"].get("info_address", zmq_info_addr) + zmq_encoding = server_settings["qserver_zmq_configuration"].get("encoding", zmq_encoding) # Read public key from the environment variable or config file. zmq_public_key = os.environ.get("QSERVER_ZMQ_PUBLIC_KEY", None) @@ -323,12 +326,13 @@ async def purge_expired_sessions_and_api_keys(): logger.info( f"Connecting to RE Manager: \nControl 0MQ socket address: {zmq_control_addr}\n" - f"Information 0MQ socket address: {zmq_info_addr}" + f"Information 0MQ socket address: {zmq_info_addr}. Encoding: {zmq_encoding!r}\n" ) RM = REManagerAPI( zmq_control_addr=zmq_control_addr, zmq_info_addr=zmq_info_addr, + zmq_encoding=zmq_encoding, zmq_public_key=zmq_public_key, request_fail_exceptions=False, status_expiration_period=0.4, # Make it smaller than default diff --git a/bluesky_httpserver/config_schemas/service_configuration.yml b/bluesky_httpserver/config_schemas/service_configuration.yml index c2502c8..57343f7 100644 --- a/bluesky_httpserver/config_schemas/service_configuration.yml +++ b/bluesky_httpserver/config_schemas/service_configuration.yml @@ -29,6 +29,10 @@ properties: The address of 0MQ info (PUB-SUB) socket of RE Manager, used to publish console output e.g. tcp://localhost:60625. Overrides the address set using QSERVER_ZMQ_INFO_ADDRESS environment variable + encoding: + type: string + description: | + The encoding for 0MQ messages. Supported values: 'json' (default) and 'msgpack'. public_key: type: string description: | diff --git a/bluesky_httpserver/database/migrations/versions/722ff4e4fcc7_add_write_scopes_to_default_roles.py b/bluesky_httpserver/database/migrations/versions/722ff4e4fcc7_add_write_scopes_to_default_roles.py index 09ff6b0..6e1fdeb 100644 --- a/bluesky_httpserver/database/migrations/versions/722ff4e4fcc7_add_write_scopes_to_default_roles.py +++ b/bluesky_httpserver/database/migrations/versions/722ff4e4fcc7_add_write_scopes_to_default_roles.py @@ -1,4 +1,4 @@ -""""Add write scopes to default Roles." +""" "Add write scopes to default Roles." Revision ID: 722ff4e4fcc7 Revises: 481830dd6c11 diff --git a/bluesky_httpserver/schemas.py b/bluesky_httpserver/schemas.py index 5da0619..c52d8f2 100644 --- a/bluesky_httpserver/schemas.py +++ b/bluesky_httpserver/schemas.py @@ -144,6 +144,7 @@ class NodeMeta(pydantic.BaseModel): class Resource(generic_model, Generic[AttributesT, ResourceLinksT, ResourceMetaT]): "A JSON API Resource" + id: Union[str, uuid.UUID] attributes: AttributesT links: Optional[ResourceLinksT] = None @@ -257,6 +258,7 @@ class Session(pydantic.BaseModel, **orm): class Principal(pydantic.BaseModel, **orm): "Represents a User or Service" + # The id field (primary key) is intentionally not exposed to the application. # It is left as an internal database concern. uuid: uuid.UUID @@ -280,6 +282,7 @@ def from_orm(cls, orm, latest_activity=None): class AllowedScopes(pydantic.BaseModel, **orm): "Returns roles and current allowed scopes for a user authenticated with API key or token" + roles: List[str] = [] scopes: List[str] = [] diff --git a/bluesky_httpserver/tests/conftest.py b/bluesky_httpserver/tests/conftest.py index d8d75e2..ec69415 100644 --- a/bluesky_httpserver/tests/conftest.py +++ b/bluesky_httpserver/tests/conftest.py @@ -4,6 +4,7 @@ import pytest import requests from bluesky_queueserver.manager.comms import zmq_single_request +from bluesky_queueserver.manager.tests.common import set_qserver_zmq_encoding # noqa: F401 from xprocess import ProcessStarter import bluesky_httpserver.server as bqss @@ -44,6 +45,8 @@ def fastapi_server_fs(xprocess): def start(http_server_host=SERVER_ADDRESS, http_server_port=SERVER_PORT, api_key=API_KEY_FOR_TESTS): class Starter(ProcessStarter): + max_read_lines = 53 + env = dict(os.environ) if api_key: env["QSERVER_HTTP_SERVER_SINGLE_USER_API_KEY"] = api_key diff --git a/bluesky_httpserver/tests/test_console_output.py b/bluesky_httpserver/tests/test_console_output.py index 55c613d..df3e877 100644 --- a/bluesky_httpserver/tests/test_console_output.py +++ b/bluesky_httpserver/tests/test_console_output.py @@ -14,6 +14,7 @@ SERVER_PORT, fastapi_server_fs, request_to_json, + set_qserver_zmq_encoding, wait_for_environment_to_be_closed, wait_for_environment_to_be_created, wait_for_manager_state_idle, @@ -155,20 +156,28 @@ def test_http_server_stream_console_output_1( """ +@pytest.mark.parametrize("zmq_encoding", (None, "json", "msgpack")) @pytest.mark.parametrize("zmq_port", (None, 60619)) -def test_http_server_console_output_1(monkeypatch, re_manager_cmd, fastapi_server_fs, zmq_port): # noqa F811 +def test_http_server_console_output_1( + monkeypatch, re_manager_cmd, fastapi_server_fs, zmq_port, zmq_encoding # noqa F811 +): """ Test for ``console_output`` API (not a streaming version). """ # Start HTTP Server if zmq_port is not None: monkeypatch.setenv("QSERVER_ZMQ_INFO_ADDRESS", f"tcp://localhost:{zmq_port}") + if zmq_encoding is not None: + monkeypatch.setenv("QSERVER_ZMQ_ENCODING", zmq_encoding) fastapi_server_fs() # Start RE Manager - params = ["--zmq-publish-console", "ON"] + params = ["--zmq-publish-console=ON"] + if zmq_encoding: + params.append(f"--zmq-encoding={zmq_encoding}") + set_qserver_zmq_encoding(monkeypatch, encoding=zmq_encoding) if zmq_port is not None: - params.extend(["--zmq-info-addr", f"tcp://*:{zmq_port}"]) + params.extend([f"--zmq-info-addr=tcp://*:{zmq_port}"]) re_manager_cmd(params) script = _script1 diff --git a/docs/source/configuration.rst b/docs/source/configuration.rst index e6c8cd8..0ed8796 100644 --- a/docs/source/configuration.rst +++ b/docs/source/configuration.rst @@ -100,6 +100,9 @@ The following environment variables are used to configure 0MQ communication sett pass the private key to RE Manager. Pass the public key to HTTP Server using ``QSERVER_ZMQ_PUBLIC_KEY`` environment variable. +- ``QSERVER_ZMQ_ENCODING`` - sets the encoding for the messages set to RE Manager over + 0MQ. Supported values: ``"json"`` (default) or ``"msgpack"``. + The same parameters can be specified by including ``qserver_zmq_configuration`` into the config file:: @@ -107,6 +110,7 @@ the config file:: control_address: tcp://localhost:60615 info_address: tcp://localhost:60625 public_key: ${PUBLIC_KEY} + encoding: json All parameters in the config file are optional and override the values passed using environment variables and the default values. The public key is typically passed using environment From 3920cc142e09dd27c696cec3fc53269475c5582d Mon Sep 17 00:00:00 2001 From: Dmitri Gavrilov Date: Sun, 4 May 2025 15:04:42 -0400 Subject: [PATCH 2/9] CI: install docker --- .github/workflows/docs.yml | 2 +- .github/workflows/docs_publish.yml | 2 +- .github/workflows/testing.yml | 23 ++++++++++++----------- 3 files changed, 14 insertions(+), 13 deletions(-) diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index 8d83c31..012a480 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -8,7 +8,7 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - python-version: [3.8] + python-version: [3.12] fail-fast: false steps: diff --git a/.github/workflows/docs_publish.yml b/.github/workflows/docs_publish.yml index 4d25df9..41ef192 100644 --- a/.github/workflows/docs_publish.yml +++ b/.github/workflows/docs_publish.yml @@ -11,7 +11,7 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - python-version: [3.8] + python-version: [3.12] fail-fast: false steps: diff --git a/.github/workflows/testing.yml b/.github/workflows/testing.yml index eeacb09..eed8a2a 100644 --- a/.github/workflows/testing.yml +++ b/.github/workflows/testing.yml @@ -8,15 +8,15 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - python-version: ["3.9", "3.10", "3.11", "3.12"] - pydantic-version: ["<2.0.0", ">=2.0.0"] - exclude: - - python-version: "3.9" - pydantic-version: "<2.0.0" - - python-version: "3.11" - pydantic-version: "<2.0.0" - - python-version: "3.12" - pydantic-version: "<2.0.0" + python-version: ["3.9", "3.10", "3.11", "3.12", "3.13"] + # pydantic-version: ["<2.0.0", ">=2.0.0"] + # exclude: + # - python-version: "3.9" + # pydantic-version: "<2.0.0" + # - python-version: "3.11" + # pydantic-version: "<2.0.0" + # - python-version: "3.12" + # pydantic-version: "<2.0.0" fail-fast: false @@ -31,6 +31,7 @@ jobs: python-version: ${{ matrix.python-version }} - name: Install dependencies run: | + sudo apt install docker sudo apt install redis # Start LDAP @@ -56,8 +57,8 @@ jobs: pip install --upgrade pip pip install . pip install -r requirements-dev.txt - pip install "pydantic${{ matrix.pydantic-version }}" - pip install bluesky==1.11.0 + # pip install "pydantic${{ matrix.pydantic-version }}" + # pip install bluesky==1.11.0 pip list - name: Test with pytest From 8533868195af1bdca3d8d7f750f49670ffc389af Mon Sep 17 00:00:00 2001 From: Dmitri Gavrilov Date: Sun, 4 May 2025 16:06:07 -0400 Subject: [PATCH 3/9] CI: set up docker --- .github/workflows/testing.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/testing.yml b/.github/workflows/testing.yml index eed8a2a..b3331a1 100644 --- a/.github/workflows/testing.yml +++ b/.github/workflows/testing.yml @@ -21,6 +21,8 @@ jobs: fail-fast: false steps: + - uses: docker-practice/actions-setup-docker@master + timeout-minutes: 12 - name: Checkout uses: actions/checkout@v4 - name: Fetch tags @@ -31,7 +33,6 @@ jobs: python-version: ${{ matrix.python-version }} - name: Install dependencies run: | - sudo apt install docker sudo apt install redis # Start LDAP From 6942e7fcf11861901673307cba910b1921b684ca Mon Sep 17 00:00:00 2001 From: Dmitri Gavrilov Date: Sun, 4 May 2025 16:24:19 -0400 Subject: [PATCH 4/9] CI: switch to Ubuntu 22.04 --- .github/workflows/testing.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/testing.yml b/.github/workflows/testing.yml index b3331a1..705a3c8 100644 --- a/.github/workflows/testing.yml +++ b/.github/workflows/testing.yml @@ -5,7 +5,7 @@ on: [push, pull_request] jobs: build: - runs-on: ubuntu-latest + runs-on: ubuntu-22.04 strategy: matrix: python-version: ["3.9", "3.10", "3.11", "3.12", "3.13"] From 624cdd2c553fa28b05daf5cb825c693687fda38b Mon Sep 17 00:00:00 2001 From: Dmitri Gavrilov Date: Sun, 4 May 2025 16:27:37 -0400 Subject: [PATCH 5/9] CI: change docker-compose to docker compose --- .github/workflows/testing.yml | 6 +++--- start_LDAP.sh | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/testing.yml b/.github/workflows/testing.yml index 705a3c8..1ada43f 100644 --- a/.github/workflows/testing.yml +++ b/.github/workflows/testing.yml @@ -5,7 +5,7 @@ on: [push, pull_request] jobs: build: - runs-on: ubuntu-22.04 + runs-on: ubuntu-latest strategy: matrix: python-version: ["3.9", "3.10", "3.11", "3.12", "3.13"] @@ -21,8 +21,8 @@ jobs: fail-fast: false steps: - - uses: docker-practice/actions-setup-docker@master - timeout-minutes: 12 + # - uses: docker-practice/actions-setup-docker@master + # timeout-minutes: 12 - name: Checkout uses: actions/checkout@v4 - name: Fetch tags diff --git a/start_LDAP.sh b/start_LDAP.sh index eca0e38..d71c662 100644 --- a/start_LDAP.sh +++ b/start_LDAP.sh @@ -4,5 +4,5 @@ set -e # Start LDAP server in docker container sudo docker pull bitnami/openldap:latest -sudo docker-compose -f .github/workflows/docker-configs/ldap-docker-compose.yml up -d +sudo docker compose -f .github/workflows/docker-configs/ldap-docker-compose.yml up -d sudo docker ps From 2738a05667711d9abeb2956e564ce32001b7bb86 Mon Sep 17 00:00:00 2001 From: Dmitri Gavrilov Date: Sun, 4 May 2025 16:31:10 -0400 Subject: [PATCH 6/9] CI: add pandas to dev requirements --- .github/workflows/testing.yml | 2 -- requirements-dev.txt | 1 + 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/testing.yml b/.github/workflows/testing.yml index 1ada43f..27a504e 100644 --- a/.github/workflows/testing.yml +++ b/.github/workflows/testing.yml @@ -21,8 +21,6 @@ jobs: fail-fast: false steps: - # - uses: docker-practice/actions-setup-docker@master - # timeout-minutes: 12 - name: Checkout uses: actions/checkout@v4 - name: Fetch tags diff --git a/requirements-dev.txt b/requirements-dev.txt index 9c02702..56a6048 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -13,6 +13,7 @@ py sphinx ipython numpydoc +pandas sphinx sphinx_rtd_theme requests From 2baf718f7c98ee35073f4f2bef24cab391a7e942 Mon Sep 17 00:00:00 2001 From: Dmitri Gavrilov Date: Sun, 4 May 2025 16:36:30 -0400 Subject: [PATCH 7/9] DOC: fixed theme config --- docs/source/conf.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/source/conf.py b/docs/source/conf.py index 29c7c93..71a5e6a 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -108,7 +108,7 @@ html_theme = 'sphinx_rtd_theme' import sphinx_rtd_theme -html_theme_path = [sphinx_rtd_theme.get_html_theme_path()] +# html_theme_path = [sphinx_rtd_theme.get_html_theme_path()] # Theme options are theme-specific and customize the look and feel of a theme # further. For a list of options available for each theme, see the From 1b4a8c2bb1d0a2c01612cc5b211cfb72062ed7b1 Mon Sep 17 00:00:00 2001 From: Dmitri Gavrilov Date: Sun, 4 May 2025 16:39:14 -0400 Subject: [PATCH 8/9] DOC: commented out matplotlib extension --- docs/source/conf.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/source/conf.py b/docs/source/conf.py index 71a5e6a..0f17058 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -40,7 +40,7 @@ 'sphinx.ext.viewcode', 'IPython.sphinxext.ipython_directive', 'IPython.sphinxext.ipython_console_highlighting', - 'matplotlib.sphinxext.plot_directive', + # 'matplotlib.sphinxext.plot_directive', 'numpydoc', ] From 93d6549d16d786af31768a623cbd1325e68e01a7 Mon Sep 17 00:00:00 2001 From: Dmitri Gavrilov Date: Sun, 4 May 2025 17:17:25 -0400 Subject: [PATCH 9/9] CI: add matplotlib to dev dependencies --- requirements-dev.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/requirements-dev.txt b/requirements-dev.txt index 56a6048..dd7212a 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -13,6 +13,7 @@ py sphinx ipython numpydoc +matplotlib pandas sphinx sphinx_rtd_theme