Integrating scicat_beamline (and 733 ingester) #99

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged

GBirkel merged 17 commits into main from 2025/12/733_scicat

Dec 24, 2025

orchestration/_tests/test_config.py

            
                      Original file line number
                      Diff line number
                      Diff line change
                  
    @@ -8,9 +8,9 @@
  
    def test_config():

        config_file = Path(__file__).parent / "test_config.yml"

        with MonkeyPatch.context():

            globus_config = read_config(config_file=config_file)

            assert globus_config

            config = read_config(config_file=config_file)

            assert config

            assert (

                globus_config["globus"]["globus_endpoints"]["test_endpoint"]["root_path"]

                config["globus"]["globus_endpoints"]["test_endpoint"]["root_path"]

                == "/data"

            )

orchestration/_tests/test_scicat.py

            
                      Original file line number
                      Diff line number
                      Diff line change
                  
    @@ -4,108 +4,19 @@
  
    import sys

    import types

    from typing import List

    from pathlib import Path

    from pytest_mock import MockFixture

    from orchestration.flows.scicat.utils import NPArrayEncoder, build_search_terms, calculate_access_controls, Issue

    from orchestration.flows.scicat.ingest import ingest_dataset_task

    from orchestration.flows.scicat.utils import Issue

    from orchestration.flows.scicat.ingest import scicat_ingest_flow

    from orchestration.flows.bl832.ingest_tomo832 import clean_email, UNKNOWN_EMAIL

    def test_clean_email_valid():

        # Remove surrounding whitespace.

        assert clean_email("  [email protected]  ") == "[email protected]"

    def test_clean_email_none():

        # Non-string input returns the default unknown email.

        assert clean_email(None) == UNKNOWN_EMAIL

    def test_clean_email_empty():

        # An empty string (or only spaces) should return the default.

        assert clean_email("   ") == UNKNOWN_EMAIL

    def test_clean_email_no_at_symbol():

        # A string without an "@" should return the default.

        assert clean_email("invalid-email") == UNKNOWN_EMAIL

    def test_clean_email_literal_none():

        # The string "NONE" (case-insensitive) should return the default.

        assert clean_email("NONE") == UNKNOWN_EMAIL

        assert clean_email("none") == UNKNOWN_EMAIL

    def test_clean_email_internal_spaces():

        # Spaces inside the email should be removed.

        # For example, " user @ example.com " should be cleaned to "[email protected]"

        assert clean_email(" user @ example.com ") == "[email protected]"

    def add_mock_requests(mock_request):

        mock_request.post("http://localhost:3000/api/v3/Users/login", json={"id": "foobar"})

        mock_request.post("http://localhost:3000/api/v3/Samples", json={"sampleId": "dataset_id"})

        mock_request.post("http://localhost:3000/api/v3/RawDatasets/replaceOrCreate", json={"pid": "42"})

        mock_request.post("http://localhost:3000/api/v3/RawDatasets/42/origdatablocks", json={"response": "random"})

    def test_np_encoder():

        test_dict = {"dont_panic": np.array([1, 2, 3], dtype=np.int8)}

        assert json.dumps(test_dict, cls=NPArrayEncoder)

        test_dict = {"dont_panic": np.array([1, 2, 3], dtype=np.float32)}

        assert json.dumps(test_dict, cls=NPArrayEncoder)

        test_dict = {"dont_panic": np.full((1, 1), np.inf)}

        encoded_np = json.loads(json.dumps(test_dict, cls=NPArrayEncoder))

        assert json.dumps(encoded_np, allow_nan=False)

    def test_build_search_terms():

        terms = build_search_terms("Time-is_an illusion. Lunchtime/2x\\so.")

        assert "time" in terms

        assert "is" in terms

        assert "an" in terms

        assert "illusion" in terms

        assert "lunchtime" in terms

        assert "2x" in terms

        assert "so" in terms

    def test_access_controls():

        username = "slartibartfast"

        # no proposal, no beamline

        access_controls = calculate_access_controls(username, None, None)

        assert access_controls["owner_group"] == "slartibartfast"

        assert access_controls["access_groups"] == []

        # proposal and no beamline

        access_controls = calculate_access_controls(username, None, "42")

        assert access_controls["owner_group"] == "42"

        # no proposal and beamline

        access_controls = calculate_access_controls(username, "10.3.1", None)

        assert access_controls["owner_group"] == "slartibartfast"

        assert "10.3.1" in access_controls["access_groups"]

        assert "slartibartfast" in access_controls["access_groups"]

        # proposal and beamline

        access_controls = calculate_access_controls(username, "10.3.1", "42")

        assert access_controls["owner_group"] == "42"

        assert "10.3.1" in access_controls["access_groups"]

        # special 8.3.2 mapping

        access_controls = calculate_access_controls(username, "bl832", "42")

        assert access_controls["owner_group"] == "42"

        assert "8.3.2" in access_controls["access_groups"]

        assert "bl832" in access_controls["access_groups"]

    def test_clean_email():

        pass

    # This code is in an indeterminate state because it is unclear whether

    # the SciCat ingest flow should be tested here, when it is already tested

    # with Prefect in the scicat_beamline package tests.

    # There may be some need for end-to-end testing where the destination SciCat

    # server is mocked or switched to the staging server, and in that case this code

    # may be useful.

    # Dummy functions and modules for external dependencies.

    def dummy_requests_post(*args, **kwargs):

    @@ -125,15 +36,24 @@ def warning(self, *args, **kwargs): pass
  
        def error(self, *args, **kwargs): pass

    def dummy_ingest(scicat_client, file_path, issues: List[Issue]):

    def dummy_ingest(

        scicat_client,

        owner_username: str,

        file_path: Path,

        thumbnail_dir: Path,

        issues: List[Issue],

    ):

        issues.clear()

        return "dummy_dataset_id"

    def test_ingest_dataset_task(mocker: MockFixture):

        return  # Temporarily disable the test.

        # Set environment variables.

        mocker.patch.dict(os.environ, {

            "SCICAT_API_URL": "http://localhost:3000/",

            "SCICAT_INGEST_URL": "http://localhost:3000/",

            "SCICAT_INGEST_USER": "test_user",

            "SCICAT_INGEST_PASSWORD": "test_password"

        })

    @@ -150,8 +70,8 @@ def test_ingest_dataset_task(mocker: MockFixture):
  
        # Inject dummy ingestor module.

        dummy_ingestor = types.ModuleType("dummy_ingestor")

        dummy_ingestor.ingest = dummy_ingest

        mocker.patch.dict(sys.modules, {"orchestration.flows.bl832.ingest_tomo832": dummy_ingestor})

        mocker.patch.dict(sys.modules, {"scicat_beamline.ingesters.als_test_ingest": dummy_ingestor})

        # Call the underlying function (.fn) of the task to bypass Prefect orchestration.

        result = ingest_dataset_task.fn("dummy_file.h5", "orchestration.flows.bl832.ingest_tomo832")

        result = scicat_ingest_flow.fn(dataset_path=Path("dummy_file.h5"), ingester_spec="bltest")

        assert result == "dummy_dataset_id"

orchestration/config.py

            
                      Original file line number
                      Diff line number
                      Diff line change
                  
    @@ -12,8 +12,8 @@ def get_config():
  
    def read_config(config_file="config.yml"):

        with open(config_file, "r") as end_file:

            globus_config = yaml.safe_load(end_file)

        return expand_environment_variables(globus_config)

            config = yaml.safe_load(end_file)

        return expand_environment_variables(config)

    def expand_environment_variables(config):

Provide feedback

Saved searches

Use saved searches to filter your results more quickly

Integrating scicat_beamline (and 733 ingester) #99

Uh oh!

Diff view

Diff view

There are no files selected for viewing

Uh oh!