From 1dd1147c2a7dd91e493caac7ef5c05bc54f25b61 Mon Sep 17 00:00:00 2001 From: Eu Pin Tien Date: Tue, 14 Jul 2026 09:50:28 +0100 Subject: [PATCH 01/17] Added new router module with an endpoint to submit SIM data processing jobs to RabbitMQ --- src/murfey/server/api/workflow_sim.py | 42 +++++++++++++++++++++++++++ src/murfey/server/main.py | 2 ++ src/murfey/util/route_manifest.yaml | 8 +++++ 3 files changed, 52 insertions(+) create mode 100644 src/murfey/server/api/workflow_sim.py diff --git a/src/murfey/server/api/workflow_sim.py b/src/murfey/server/api/workflow_sim.py new file mode 100644 index 000000000..7e1523881 --- /dev/null +++ b/src/murfey/server/api/workflow_sim.py @@ -0,0 +1,42 @@ +import logging +from pathlib import Path + +from fastapi import APIRouter, Depends +from pydantic import BaseModel + +from murfey.server import _transport_object +from murfey.server.api.auth import validate_instrument_token + +logger = logging.getLogger("murfey.server.api.workflow_fib") + +router = APIRouter( + prefix="/workflow/sim", + dependencies=[Depends(validate_instrument_token)], + tags=["Workflows: CryoSIM"], +) + + +class SIMDataFile(BaseModel): + file: Path + + +@router.post("/sessions/{session_id}/process_data") +def request_sim_processing(session_id: int, sim_data: SIMDataFile): + if _transport_object is None: + logger.error("No TransportManager object was set up") + return None + + # Construct message and submit it to 'processing_recipe' + logger.info(f"Submitting request to process the cryoSIM file {sim_data.file}") + recipe = { + # Placeholder; fields will be populated once service is set up + "recipes": ["sim-process-data"], + "parameters": { + # Job parameters + "file": f"{str(sim_data.file)}", + "feedback_queue": _transport_object.feedback_queue, + }, + } + _transport_object.send( + queue="processing_recipe", message=recipe, new_connection=True + ) diff --git a/src/murfey/server/main.py b/src/murfey/server/main.py index c14be9f2c..0313a481b 100644 --- a/src/murfey/server/main.py +++ b/src/murfey/server/main.py @@ -27,6 +27,7 @@ import murfey.server.api.workflow import murfey.server.api.workflow_clem import murfey.server.api.workflow_fib +import murfey.server.api.workflow_sim import murfey.server.api.workflow_sxt from murfey.server import template_files from murfey.util.config import get_security_config @@ -100,6 +101,7 @@ class Settings(BaseSettings): app.include_router(murfey.server.api.workflow.tomo_router) app.include_router(murfey.server.api.workflow_clem.router) app.include_router(murfey.server.api.workflow_fib.router) +app.include_router(murfey.server.api.workflow_sim.router) app.include_router(murfey.server.api.workflow_sxt.router) app.include_router(murfey.server.api.prometheus.router) diff --git a/src/murfey/util/route_manifest.yaml b/src/murfey/util/route_manifest.yaml index e80e1237b..bb8641d9e 100644 --- a/src/murfey/util/route_manifest.yaml +++ b/src/murfey/util/route_manifest.yaml @@ -1445,6 +1445,14 @@ murfey.server.api.workflow_fib.router: type: int methods: - POST +murfey.server.api.workflow_sim.router: + - path: /workflow/sim/sessions/{session_id}/process_data + function: request_sim_processing + path_params: + - name: session_id + type: int + methods: + - POST murfey.server.api.workflow_sxt.router: - path: /workflow/sxt/visits/{visit_name}/sessions/{session_id}/sxt_tilt_series function: process_sxt_tilt_series From 9ff1ab4adfe53acf5f16ab1a7ccff3833107f5fe Mon Sep 17 00:00:00 2001 From: Eu Pin Tien Date: Tue, 14 Jul 2026 10:15:11 +0100 Subject: [PATCH 02/17] Added CryoSIM context to list of Murfey context entry points --- pyproject.toml | 1 + 1 file changed, 1 insertion(+) diff --git a/pyproject.toml b/pyproject.toml index 506cf8358..51defe6fe 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -101,6 +101,7 @@ GitHub = "https://github.com/DiamondLightSource/python-murfey" AtlasContext = "murfey.client.contexts.atlas:AtlasContext" CLEMContext = "murfey.client.contexts.clem:CLEMContext" FIBContext = "murfey.client.contexts.fib:FIBContext" +SIMContext = "murfey.client.contexts.sim:SIMContext" SPAContext = "murfey.client.contexts.spa:SPAContext" SPAMetadataContext = "murfey.client.contexts.spa_metadata:SPAMetadataContext" SXTContext = "murfey.client.contexts.sxt:SXTContext" From e054490a3d1d5d230a350cddb564a15d33a6685c Mon Sep 17 00:00:00 2001 From: Eu Pin Tien Date: Tue, 14 Jul 2026 10:15:52 +0100 Subject: [PATCH 03/17] Added new utility module containing shared variables and functions used by the CryoSIM workflow --- src/murfey/util/sim.py | 7 +++++++ 1 file changed, 7 insertions(+) create mode 100644 src/murfey/util/sim.py diff --git a/src/murfey/util/sim.py b/src/murfey/util/sim.py new file mode 100644 index 000000000..abf75d12d --- /dev/null +++ b/src/murfey/util/sim.py @@ -0,0 +1,7 @@ +SIM_DATA_SUFFIXES = ( + # SIM raw data files have their stems ending with these suffixes + "_BR", + "_BFR", + "_GR", + "_GFR", +) From 020268f463968fa0c7968cf7ade326d9d4cc9ac8 Mon Sep 17 00:00:00 2001 From: Eu Pin Tien Date: Tue, 14 Jul 2026 10:23:18 +0100 Subject: [PATCH 04/17] Added logic to the Analyser to identify and set CryoSIM context --- src/murfey/client/analyser.py | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/src/murfey/client/analyser.py b/src/murfey/client/analyser.py index 9d9182f47..ac6513790 100644 --- a/src/murfey/client/analyser.py +++ b/src/murfey/client/analyser.py @@ -23,6 +23,7 @@ from murfey.util.client import Observer, get_machine_config_client from murfey.util.mdoc import get_block from murfey.util.models import ProcessingParametersSPA, ProcessingParametersTomo +from murfey.util.sim import SIM_DATA_SUFFIXES logger = logging.getLogger("murfey.client.analyser") @@ -215,6 +216,23 @@ def _find_context(self, file_path: Path) -> bool: ) return True + # ----------------------------------------------------------------------------- + # SIM workflow checks + # ----------------------------------------------------------------------------- + if ( + # CryoSIM raw data files have no extension, and end with specific suffixes + not file_path.suffix and file_path.stem.endswith(SIM_DATA_SUFFIXES) + ): + if (context := _get_context("SIMContext")) is None: + return False + self._context = context.load()( + "sim", + self._basepath, + self._murfey_config, + self._token, + ) + return True + # ----------------------------------------------------------------------------- # SXT workflow checks # ----------------------------------------------------------------------------- From 9ac66a261860bf8e0c1c48d5344ef9d57f512559 Mon Sep 17 00:00:00 2001 From: Eu Pin Tien Date: Tue, 14 Jul 2026 10:24:53 +0100 Subject: [PATCH 05/17] Added logic to the CryoSIM context to identify raw data files and submit them for processing --- src/murfey/client/contexts/sim.py | 69 +++++++++++++++++++++++++++++++ 1 file changed, 69 insertions(+) diff --git a/src/murfey/client/contexts/sim.py b/src/murfey/client/contexts/sim.py index 699b9451e..0779dda9d 100644 --- a/src/murfey/client/contexts/sim.py +++ b/src/murfey/client/contexts/sim.py @@ -3,10 +3,40 @@ from murfey.client.context import Context from murfey.client.instance_environment import MurfeyInstanceEnvironment +from murfey.util.client import capture_post +from murfey.util.sim import SIM_DATA_SUFFIXES logger = logging.getLogger("murfey.client.contexts.sim") +def _get_source(file_path: Path, environment: MurfeyInstanceEnvironment) -> Path | None: + """ + Returns the Path of the file on the client PC. + """ + for s in environment.sources: + if file_path.is_relative_to(s): + return s + return None + + +def _file_transferred_to( + environment: MurfeyInstanceEnvironment, + source: Path, + file_path: Path, + rsync_basepath: Path, +) -> Path | None: + """ + Returns the Path of the transferred file on the DLS file system. + """ + # Construct destination path + base_destination = rsync_basepath / Path(environment.default_destinations[source]) + # Add visit number to the path if it's not present in default destination + if environment.visit not in environment.default_destinations[source]: + base_destination = base_destination / environment.visit + destination = base_destination / file_path.relative_to(source) + return destination + + class SIMContext(Context): def __init__( self, @@ -25,4 +55,43 @@ def post_transfer( environment: MurfeyInstanceEnvironment | None = None, **kwargs, ): + super().post_transfer(transferred_file, environment=environment, **kwargs) + if environment is None: + logger.warning("No environment passed in") + return None + + # Look for raw data files + # These have no extensions, and end with one of the listed suffixes + if not transferred_file.suffix and transferred_file.stem.endswith( + SIM_DATA_SUFFIXES + ): + source = _get_source(transferred_file, environment) + if source is None: + logger.warning(f"No source found for file {transferred_file}") + return None + destination_file = _file_transferred_to( + environment=environment, + source=source, + file_path=transferred_file, + rsync_basepath=Path(self._machine_config.get("rsync_basepath", "")), + ) + if destination_file is None: + logger.warning( + f"Could not find destination file path for {transferred_file.name!r}" + ) + return None + capture_post( + base_url=str(environment.url.geturl()), + router_name="workflow_sim.router", + function_name="request_sim_processing", + token=self._token, + instrument_name=environment.instrument_name, + data={ + "file": f"{destination_file}", + }, + # Endpoint kwargs + session_id=environment.murfey_session, + ) + return None + return None From 3eb39ee85481ab763ab445b183b9a1cb0db57de3 Mon Sep 17 00:00:00 2001 From: Eu Pin Tien Date: Tue, 14 Jul 2026 10:28:25 +0100 Subject: [PATCH 06/17] Forgot to rename logger for cryoSIM API module --- src/murfey/server/api/workflow_sim.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/murfey/server/api/workflow_sim.py b/src/murfey/server/api/workflow_sim.py index 7e1523881..ffd785730 100644 --- a/src/murfey/server/api/workflow_sim.py +++ b/src/murfey/server/api/workflow_sim.py @@ -7,7 +7,7 @@ from murfey.server import _transport_object from murfey.server.api.auth import validate_instrument_token -logger = logging.getLogger("murfey.server.api.workflow_fib") +logger = logging.getLogger("murfey.server.api.workflow_sim") router = APIRouter( prefix="/workflow/sim", From ca1409fe9edd46fa3adf80d2c8afd8c58344ec81 Mon Sep 17 00:00:00 2001 From: Eu Pin Tien Date: Tue, 14 Jul 2026 11:20:06 +0100 Subject: [PATCH 07/17] Added test for the 'request_sim_processing' API endpoint --- tests/server/api/test_workflow_sim.py | 51 +++++++++++++++++++++++++++ 1 file changed, 51 insertions(+) create mode 100644 tests/server/api/test_workflow_sim.py diff --git a/tests/server/api/test_workflow_sim.py b/tests/server/api/test_workflow_sim.py new file mode 100644 index 000000000..f37715f2e --- /dev/null +++ b/tests/server/api/test_workflow_sim.py @@ -0,0 +1,51 @@ +from pathlib import Path +from unittest.mock import MagicMock + +import pytest +from pytest_mock import MockerFixture + +from murfey.server.api.workflow_sim import SIMDataFile, request_sim_processing + + +@pytest.mark.parametrize("has_transport_object", (True, False)) +def test_request_sim_processing( + mocker: MockerFixture, tmp_path: Path, has_transport_object: bool +): + # Set up the variables + session_id = 1 + sim_data = SIMDataFile(**{"file": str(tmp_path / "dummy")}) + + # Mock the logger + mock_logger = mocker.patch("murfey.server.api.workflow_sim.logger") + + # Mock the transport object + if has_transport_object: + mock_transport_object = MagicMock() + mock_transport_object.feedback_queue = "dummy" + mocker.patch( + "murfey.server.api.workflow_sim._transport_object", + mock_transport_object, + ) + else: + mocker.patch( + "murfey.server.api.workflow_sim._transport_object", + None, + ) + + # Run the function and check that the expected calls were made + request_sim_processing( + session_id=session_id, + sim_data=sim_data, + ) + + # Check that the expected calls were made + if has_transport_object: + recipe = { + "recipes": ["sim-process-data"], + "parameters": {"file": f"{sim_data.file}", "feedback_queue": "dummy"}, + } + mock_transport_object.send.assert_called_with( + queue="processing_recipe", message=recipe, new_connection=True + ) + else: + mock_logger.error.assert_called_with("No TransportManager object was set up") From f424ceea7dc1ec7e7ca534303767463bda4c5f87 Mon Sep 17 00:00:00 2001 From: Eu Pin Tien Date: Tue, 14 Jul 2026 11:28:34 +0100 Subject: [PATCH 08/17] Updated cryoSIM raw data suffixes --- src/murfey/util/sim.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/murfey/util/sim.py b/src/murfey/util/sim.py index abf75d12d..1667a9aa6 100644 --- a/src/murfey/util/sim.py +++ b/src/murfey/util/sim.py @@ -4,4 +4,8 @@ "_BFR", "_GR", "_GFR", + "_BR_FL", + "_BFR_FL", + "_GR_FL", + "_GFR_FL", ) From 3fc06653e9c38d984c79f2161ab72b211e19065d Mon Sep 17 00:00:00 2001 From: Eu Pin Tien Date: Tue, 14 Jul 2026 11:40:34 +0100 Subject: [PATCH 09/17] Added tests for the Analyser for the cryoSIM workflow --- tests/client/test_analyser.py | 39 +++++++++++++++++++++++++++++++++++ 1 file changed, 39 insertions(+) diff --git a/tests/client/test_analyser.py b/tests/client/test_analyser.py index 5deda832b..912dbed5d 100644 --- a/tests/client/test_analyser.py +++ b/tests/client/test_analyser.py @@ -36,6 +36,16 @@ "visit/maps/visit/LayersData/Layer/Electron Snapshot/Electron Snapshot.tiff", "visit/maps/visit/LayersData/Layer/Electron Snapshot (2)/Electron Snapshot (2).tiff", ], + "SIMContext": [ + "visit/raw/SR002_G1/20260707_112417_SR002G1_F1F_BR", + "visit/raw/SR002_G1/20260707_112417_SR002G1_F1F_BFR", + "visit/raw/44drug_G2/20260703_114348_44drug_G2_E2DR_GR", + "visit/raw/44drug_G2/20260703_113142_44drug_G2_E2DR_GFR", + "visit/raw/SR002_G1/20260707_112417_SR002G1_F1F_BR_FL", + "visit/raw/SR002_G1/20260707_112417_SR002G1_F1F_BFR_FL", + "visit/raw/44drug_G2/20260703_114348_44drug_G2_E2DR_GR_FL", + "visit/raw/44drug_G2/20260703_113142_44drug_G2_E2DR_GFR_FL", + ], "SXTContext": [ "visit/tomo__tag_ROI10_area1_angle-60to60@1.5_1sec_251p.txrm", "visit/X-ray_mosaic_ROI2.xrm", @@ -263,6 +273,35 @@ def test_analyse_fib( assert mock_post_transfer.call_count == len(test_files) +def test_analyse_sim( + mocker: MockerFixture, + tmp_path: Path, +): + # Load the example files corresponding to the SIM workflow + test_files = [ + file + for context, file_list in example_files.items() + for file in file_list + if context == "SIMContext" + ] + + # Mock the 'post_transfer' class function + mock_post_transfer = mocker.patch.object(Analyser, "post_transfer") + spy_find_context = mocker.spy(Analyser, "_find_context") + + # Initialise the Analyser + analyser = Analyser(tmp_path, "") + for file in test_files: + analyser._analyse(tmp_path / file) + + # "_find_context" should be called only once + assert spy_find_context.call_count == 1 + assert analyser._context is not None and "SIMContext" in str(analyser._context) + + # "_post_transfer" should be called on every one of these files + assert mock_post_transfer.call_count == len(test_files) + + def test_analyse_sxt( mocker: MockerFixture, tmp_path: Path, From 008e62ba0fabff809670273cdfff5e8e79a4a238 Mon Sep 17 00:00:00 2001 From: Eu Pin Tien Date: Tue, 14 Jul 2026 12:19:16 +0100 Subject: [PATCH 10/17] Updated tests for the cryoSIM context to reflect updated functionality --- tests/client/contexts/test_sim.py | 179 ++++++++++++++++++++++++++---- 1 file changed, 157 insertions(+), 22 deletions(-) diff --git a/tests/client/contexts/test_sim.py b/tests/client/contexts/test_sim.py index d0e8beb38..a2ab7c14c 100644 --- a/tests/client/contexts/test_sim.py +++ b/tests/client/contexts/test_sim.py @@ -1,6 +1,78 @@ from pathlib import Path +from unittest import mock +from unittest.mock import MagicMock -from murfey.client.contexts.sim import SIMContext +import pytest +from pytest_mock import MockerFixture + +from murfey.client.contexts.sim import SIMContext, _file_transferred_to, _get_source + +visit_name = "cm12345-6" +instrument_name = "sim" +session_id = 1 + + +@pytest.fixture +def visit_dir(tmp_path: Path): + return tmp_path / visit_name + + +@pytest.fixture +def sim_data(visit_dir: Path): + file_list = [] + for path in [ + "raw/SR002_G1/20260707_112417_SR002G1_F1F_BR", + "raw/SR002_G1/20260707_112417_SR002G1_F1F_BFR", + "raw/44drug_G2/20260703_114348_44drug_G2_E2DR_GR", + "raw/44drug_G2/20260703_113142_44drug_G2_E2DR_GFR", + "raw/SR002_G1/20260707_112417_SR002G1_F1F_BR_FL", + "raw/SR002_G1/20260707_112417_SR002G1_F1F_BFR_FL", + "raw/44drug_G2/20260703_114348_44drug_G2_E2DR_GR_FL", + "raw/44drug_G2/20260703_113142_44drug_G2_E2DR_GFR_FL", + ]: + file = visit_dir / path + file.parent.mkdir(parents=True, exist_ok=True) + file.touch(exist_ok=True) + file_list.append(file) + return file_list + + +def test_get_source( + tmp_path: Path, + visit_dir: Path, + sim_data: list[Path], +): + # Mock the MurfeyInstanceEnvironment + mock_environment = MagicMock() + mock_environment.sources = [ + visit_dir, + tmp_path / "another_dir", + ] + # Check that the correct source directory is found + for file in sim_data: + assert _get_source(file, mock_environment) == visit_dir + + +def test_file_transferred_to( + tmp_path: Path, + visit_dir: Path, + sim_data: list[Path], +): + # Mock the environment + mock_environment = MagicMock() + mock_environment.default_destinations = {visit_dir: "current_year"} + mock_environment.visit = visit_name + + # Iterate across the FIB files to compare against + destination_dir = tmp_path / "sim" / "data" / "current_year" / visit_name + for file in sim_data: + # Work out what the expected destination will be + assert _file_transferred_to( + environment=mock_environment, + source=visit_dir, + file_path=file, + rsync_basepath=tmp_path / "sim" / "data", + ) == destination_dir / file.relative_to(visit_dir) def test_sim_context_initialises(tmp_path: Path): @@ -20,33 +92,96 @@ def test_sim_context_initialises(tmp_path: Path): assert context.name == "SIMContext" +@pytest.mark.parametrize( + "test_params", + ( # Has environment | Has source | Has destination + # Success case + (True, True, True), + # Fail cases + (True, True, False), # No destination + (True, False, True), # No source + (False, True, True), # No environment + ), +) def test_post_transfer( + mocker: MockerFixture, + test_params: tuple[bool, bool, bool], tmp_path: Path, + visit_dir: Path, + sim_data: list[Path], ): - """ - NOTE: This is just a basic test for coverage purposes, and will be rewritten - as the SIMContext logic evolves and matures. - """ - # Create a dummy file - base_path = tmp_path - visit_dir = base_path / "visit" - test_file = visit_dir / "raw" / "dummy.txt" - test_file.parent.mkdir(parents=True, exist_ok=True) - test_file.touch(exist_ok=True) + # Unpack test params + use_env, has_src, has_dst = test_params - # Create other mock variables - machine_config = {"dummy": "dummy"} + # Mock the environment + mock_environment = None + if use_env: + mock_environment = MagicMock() + mock_environment.visit = visit_name + mock_environment.instrument_name = instrument_name + mock_environment.murfey_session = session_id + + # Mock the logger to check if specific logs are triggered + mock_logger = mocker.patch("murfey.client.contexts.sim.logger") + + # Iterate across the FIB files to compare against + destination_dir = tmp_path / "sim" / "data" / "current_year" / visit_name + destination_files = [ + destination_dir / file.relative_to(visit_dir) for file in sim_data + ] + + # Mock the functions used in 'post_transfer' + mock_get_source = mocker.patch("murfey.client.contexts.sim._get_source") + mock_get_source.return_value = tmp_path if has_src else None + + mock_file_transferred_to = mocker.patch( + "murfey.client.contexts.sim._file_transferred_to" + ) + if has_dst: + mock_file_transferred_to.side_effect = destination_files + else: + mock_file_transferred_to.return_value = None + mock_capture_post = mocker.patch("murfey.client.contexts.sim.capture_post") + + # Initialise the SIMContext + basepath = tmp_path context = SIMContext( - "sim", - basepath=base_path, - machine_config=machine_config, + acquisition_software="sim", + basepath=basepath, + machine_config={}, token="dummy", ) - assert ( - context.post_transfer( - transferred_file=test_file, - environment=None, + for file in sim_data: + context.post_transfer(file, environment=mock_environment) + if not use_env: + mock_logger.warning.assert_called_with("No environment passed in") + elif not has_src: + mock_logger.warning.assert_called_with(f"No source found for file {file}") + elif not has_dst: + mock_logger.warning.assert_called_with( + f"Could not find destination file path for {file.name!r}" ) - is None - ) + else: + mock_get_source.assert_called_with(file, mock_environment) + mock_file_transferred_to.assert_called_with( + environment=mock_environment, + source=basepath, + file_path=file, + rsync_basepath=Path(""), + ) + + assert mock_capture_post.call_count == len(sim_data) + for dst in destination_files: + mock_capture_post.assert_any_call( + base_url=mock.ANY, + router_name="workflow_sim.router", + function_name="request_sim_processing", + token=context._token, + instrument_name=instrument_name, + data={ + "file": f"{dst}", + }, + # Endpoint kwargs + session_id=session_id, + ) From 7a783a73dc087355cb03efc50ae977924c8aefb0 Mon Sep 17 00:00:00 2001 From: Eu Pin Tien Date: Tue, 14 Jul 2026 12:26:13 +0100 Subject: [PATCH 11/17] Sanitise file path --- src/murfey/server/api/workflow_sim.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/murfey/server/api/workflow_sim.py b/src/murfey/server/api/workflow_sim.py index ffd785730..6288bab03 100644 --- a/src/murfey/server/api/workflow_sim.py +++ b/src/murfey/server/api/workflow_sim.py @@ -6,6 +6,7 @@ from murfey.server import _transport_object from murfey.server.api.auth import validate_instrument_token +from murfey.util import sanitise_path logger = logging.getLogger("murfey.server.api.workflow_sim") @@ -27,7 +28,9 @@ def request_sim_processing(session_id: int, sim_data: SIMDataFile): return None # Construct message and submit it to 'processing_recipe' - logger.info(f"Submitting request to process the cryoSIM file {sim_data.file}") + logger.info( + f"Submitting request to process the cryoSIM file {sanitise_path(sim_data.file)}" + ) recipe = { # Placeholder; fields will be populated once service is set up "recipes": ["sim-process-data"], From 9d1395ed61e7311082b79f92ac81e4f6d10b7cac Mon Sep 17 00:00:00 2001 From: Eu Pin Tien Date: Tue, 14 Jul 2026 14:23:24 +0100 Subject: [PATCH 12/17] Put the file endings in the Analyser and SIMContext files directly; Analyser should determine context using bright field files as well, whereas 'capture_post' should only be triggered on fluorescent ones --- src/murfey/client/analyser.py | 18 ++++++++++++++++-- src/murfey/client/contexts/sim.py | 14 ++++++++++++-- src/murfey/util/sim.py | 11 ----------- 3 files changed, 28 insertions(+), 15 deletions(-) delete mode 100644 src/murfey/util/sim.py diff --git a/src/murfey/client/analyser.py b/src/murfey/client/analyser.py index ac6513790..5046fd9dd 100644 --- a/src/murfey/client/analyser.py +++ b/src/murfey/client/analyser.py @@ -23,7 +23,6 @@ from murfey.util.client import Observer, get_machine_config_client from murfey.util.mdoc import get_block from murfey.util.models import ProcessingParametersSPA, ProcessingParametersTomo -from murfey.util.sim import SIM_DATA_SUFFIXES logger = logging.getLogger("murfey.client.analyser") @@ -221,7 +220,22 @@ def _find_context(self, file_path: Path) -> bool: # ----------------------------------------------------------------------------- if ( # CryoSIM raw data files have no extension, and end with specific suffixes - not file_path.suffix and file_path.stem.endswith(SIM_DATA_SUFFIXES) + not file_path.suffix + and file_path.stem.endswith( + ( + # Bright field + "_BF", + # Fluorescent + "_BR", + "_BFR", + "_GR", + "_GFR", + "_BR_FL", + "_BFR_FL", + "_GR_FL", + "_GFR_FL", + ) + ) ): if (context := _get_context("SIMContext")) is None: return False diff --git a/src/murfey/client/contexts/sim.py b/src/murfey/client/contexts/sim.py index 0779dda9d..fb8796e41 100644 --- a/src/murfey/client/contexts/sim.py +++ b/src/murfey/client/contexts/sim.py @@ -4,7 +4,6 @@ from murfey.client.context import Context from murfey.client.instance_environment import MurfeyInstanceEnvironment from murfey.util.client import capture_post -from murfey.util.sim import SIM_DATA_SUFFIXES logger = logging.getLogger("murfey.client.contexts.sim") @@ -63,7 +62,17 @@ def post_transfer( # Look for raw data files # These have no extensions, and end with one of the listed suffixes if not transferred_file.suffix and transferred_file.stem.endswith( - SIM_DATA_SUFFIXES + ( + # Fluorescent SIM raw data files end as follows + "_BR", + "_BFR", + "_GR", + "_GFR", + "_BR_FL", + "_BFR_FL", + "_GR_FL", + "_GFR_FL", + ) ): source = _get_source(transferred_file, environment) if source is None: @@ -80,6 +89,7 @@ def post_transfer( f"Could not find destination file path for {transferred_file.name!r}" ) return None + # Submit fluorescent raw data files for processing capture_post( base_url=str(environment.url.geturl()), router_name="workflow_sim.router", diff --git a/src/murfey/util/sim.py b/src/murfey/util/sim.py deleted file mode 100644 index 1667a9aa6..000000000 --- a/src/murfey/util/sim.py +++ /dev/null @@ -1,11 +0,0 @@ -SIM_DATA_SUFFIXES = ( - # SIM raw data files have their stems ending with these suffixes - "_BR", - "_BFR", - "_GR", - "_GFR", - "_BR_FL", - "_BFR_FL", - "_GR_FL", - "_GFR_FL", -) From c5219b4bf3fa88df169241d219a9359a0066207d Mon Sep 17 00:00:00 2001 From: Eu Pin Tien Date: Tue, 14 Jul 2026 14:24:15 +0100 Subject: [PATCH 13/17] Updated tests to reflect new behaviour --- tests/client/contexts/test_sim.py | 54 ++++++++++++++++++------------- tests/client/test_analyser.py | 3 ++ 2 files changed, 34 insertions(+), 23 deletions(-) diff --git a/tests/client/contexts/test_sim.py b/tests/client/contexts/test_sim.py index a2ab7c14c..a7342789a 100644 --- a/tests/client/contexts/test_sim.py +++ b/tests/client/contexts/test_sim.py @@ -21,6 +21,9 @@ def visit_dir(tmp_path: Path): def sim_data(visit_dir: Path): file_list = [] for path in [ + # Bright field + "raw/CtrlApr_G2/20260703_132856_CtrlApr_G2_F3A_BF", + # Fluorescent "raw/SR002_G1/20260707_112417_SR002G1_F1F_BR", "raw/SR002_G1/20260707_112417_SR002G1_F1F_BFR", "raw/44drug_G2/20260703_114348_44drug_G2_E2DR_GR", @@ -127,7 +130,9 @@ def test_post_transfer( # Iterate across the FIB files to compare against destination_dir = tmp_path / "sim" / "data" / "current_year" / visit_name destination_files = [ - destination_dir / file.relative_to(visit_dir) for file in sim_data + destination_dir / file.relative_to(visit_dir) + for file in sim_data + if not file.stem.endswith("_BF") ] # Mock the functions used in 'post_transfer' @@ -152,6 +157,7 @@ def test_post_transfer( machine_config={}, token="dummy", ) + # Pass the list of files through for file in sim_data: context.post_transfer(file, environment=mock_environment) if not use_env: @@ -163,25 +169,27 @@ def test_post_transfer( f"Could not find destination file path for {file.name!r}" ) else: - mock_get_source.assert_called_with(file, mock_environment) - mock_file_transferred_to.assert_called_with( - environment=mock_environment, - source=basepath, - file_path=file, - rsync_basepath=Path(""), - ) - - assert mock_capture_post.call_count == len(sim_data) - for dst in destination_files: - mock_capture_post.assert_any_call( - base_url=mock.ANY, - router_name="workflow_sim.router", - function_name="request_sim_processing", - token=context._token, - instrument_name=instrument_name, - data={ - "file": f"{dst}", - }, - # Endpoint kwargs - session_id=session_id, - ) + for src, dst in zip(sim_data, [Path(""), *destination_files]): + if src.stem.endswith("_BF"): + continue + else: + mock_get_source.assert_any_call(src, mock_environment) + mock_file_transferred_to.assert_any_call( + environment=mock_environment, + source=basepath, + file_path=src, + rsync_basepath=Path(""), + ) + mock_capture_post.assert_any_call( + base_url=mock.ANY, + router_name="workflow_sim.router", + function_name="request_sim_processing", + token=context._token, + instrument_name=instrument_name, + data={ + "file": f"{dst}", + }, + # Endpoint kwargs + session_id=session_id, + ) + assert mock_capture_post.call_count == len(sim_data) - 1 diff --git a/tests/client/test_analyser.py b/tests/client/test_analyser.py index 912dbed5d..0694c0569 100644 --- a/tests/client/test_analyser.py +++ b/tests/client/test_analyser.py @@ -37,6 +37,9 @@ "visit/maps/visit/LayersData/Layer/Electron Snapshot (2)/Electron Snapshot (2).tiff", ], "SIMContext": [ + # Bright field + "visit/raw/CtrlApr_G2/20260703_132856_CtrlApr_G2_F3A_BF", + # Fluorescent "visit/raw/SR002_G1/20260707_112417_SR002G1_F1F_BR", "visit/raw/SR002_G1/20260707_112417_SR002G1_F1F_BFR", "visit/raw/44drug_G2/20260703_114348_44drug_G2_E2DR_GR", From 442073e63c99859b2d3ac13704c536f911c1e4c5 Mon Sep 17 00:00:00 2001 From: Eu Pin Tien Date: Wed, 15 Jul 2026 06:15:05 +0100 Subject: [PATCH 14/17] Add client-side log when requesting for processing --- src/murfey/client/contexts/sim.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/murfey/client/contexts/sim.py b/src/murfey/client/contexts/sim.py index fb8796e41..4b2a711f9 100644 --- a/src/murfey/client/contexts/sim.py +++ b/src/murfey/client/contexts/sim.py @@ -89,7 +89,9 @@ def post_transfer( f"Could not find destination file path for {transferred_file.name!r}" ) return None + # Submit fluorescent raw data files for processing + logger.info(f"Requesting processing for {transferred_file.name!r}") capture_post( base_url=str(environment.url.geturl()), router_name="workflow_sim.router", From 9e465ed9a537ff61b6def1774cd538c6f0c19771 Mon Sep 17 00:00:00 2001 From: Eu Pin Tien Date: Wed, 15 Jul 2026 08:07:03 +0100 Subject: [PATCH 15/17] Disable actual submission of message to 'processing_recipe' queue --- src/murfey/server/api/workflow_sim.py | 10 ++++++---- tests/server/api/test_workflow_sim.py | 13 ++++++++++--- 2 files changed, 16 insertions(+), 7 deletions(-) diff --git a/src/murfey/server/api/workflow_sim.py b/src/murfey/server/api/workflow_sim.py index 6288bab03..83e9e2720 100644 --- a/src/murfey/server/api/workflow_sim.py +++ b/src/murfey/server/api/workflow_sim.py @@ -32,14 +32,16 @@ def request_sim_processing(session_id: int, sim_data: SIMDataFile): f"Submitting request to process the cryoSIM file {sanitise_path(sim_data.file)}" ) recipe = { - # Placeholder; fields will be populated once service is set up "recipes": ["sim-process-data"], "parameters": { # Job parameters + "session_id": session_id, "file": f"{str(sim_data.file)}", "feedback_queue": _transport_object.feedback_queue, }, } - _transport_object.send( - queue="processing_recipe", message=recipe, new_connection=True - ) + logger.debug(f"Will submit the following message to 'processing_recipe':\n{recipe}") + # Disabled for now; will submit message once recipe and service have been set up + # _transport_object.send( + # queue="processing_recipe", message=recipe, new_connection=True + # ) diff --git a/tests/server/api/test_workflow_sim.py b/tests/server/api/test_workflow_sim.py index f37715f2e..2da84efd5 100644 --- a/tests/server/api/test_workflow_sim.py +++ b/tests/server/api/test_workflow_sim.py @@ -42,10 +42,17 @@ def test_request_sim_processing( if has_transport_object: recipe = { "recipes": ["sim-process-data"], - "parameters": {"file": f"{sim_data.file}", "feedback_queue": "dummy"}, + "parameters": { + "session_id": session_id, + "file": f"{sim_data.file}", + "feedback_queue": "dummy", + }, } - mock_transport_object.send.assert_called_with( - queue="processing_recipe", message=recipe, new_connection=True + mock_logger.debug.assert_called_with( + f"Will submit the following message to 'processing_recipe':\n{recipe}" ) + # mock_transport_object.send.assert_called_with( + # queue="processing_recipe", message=recipe, new_connection=True + # ) else: mock_logger.error.assert_called_with("No TransportManager object was set up") From f28374767ad54f5b1bca43926e6eb8b59e51f48e Mon Sep 17 00:00:00 2001 From: Eu Pin Tien Date: Thu, 16 Jul 2026 03:10:27 +0100 Subject: [PATCH 16/17] Switch to using shared '_file_transferred_to' and '_get_source' functions, and updated cryoSIM tests to reflect this --- src/murfey/client/contexts/sim.py | 35 +------------------------------ tests/client/contexts/test_sim.py | 26 ++++++++--------------- 2 files changed, 10 insertions(+), 51 deletions(-) diff --git a/src/murfey/client/contexts/sim.py b/src/murfey/client/contexts/sim.py index 4b2a711f9..81f6bd83f 100644 --- a/src/murfey/client/contexts/sim.py +++ b/src/murfey/client/contexts/sim.py @@ -1,41 +1,13 @@ import logging from pathlib import Path -from murfey.client.context import Context +from murfey.client.context import Context, _file_transferred_to, _get_source from murfey.client.instance_environment import MurfeyInstanceEnvironment from murfey.util.client import capture_post logger = logging.getLogger("murfey.client.contexts.sim") -def _get_source(file_path: Path, environment: MurfeyInstanceEnvironment) -> Path | None: - """ - Returns the Path of the file on the client PC. - """ - for s in environment.sources: - if file_path.is_relative_to(s): - return s - return None - - -def _file_transferred_to( - environment: MurfeyInstanceEnvironment, - source: Path, - file_path: Path, - rsync_basepath: Path, -) -> Path | None: - """ - Returns the Path of the transferred file on the DLS file system. - """ - # Construct destination path - base_destination = rsync_basepath / Path(environment.default_destinations[source]) - # Add visit number to the path if it's not present in default destination - if environment.visit not in environment.default_destinations[source]: - base_destination = base_destination / environment.visit - destination = base_destination / file_path.relative_to(source) - return destination - - class SIMContext(Context): def __init__( self, @@ -84,11 +56,6 @@ def post_transfer( file_path=transferred_file, rsync_basepath=Path(self._machine_config.get("rsync_basepath", "")), ) - if destination_file is None: - logger.warning( - f"Could not find destination file path for {transferred_file.name!r}" - ) - return None # Submit fluorescent raw data files for processing logger.info(f"Requesting processing for {transferred_file.name!r}") diff --git a/tests/client/contexts/test_sim.py b/tests/client/contexts/test_sim.py index a7342789a..80acf2d93 100644 --- a/tests/client/contexts/test_sim.py +++ b/tests/client/contexts/test_sim.py @@ -5,7 +5,8 @@ import pytest from pytest_mock import MockerFixture -from murfey.client.contexts.sim import SIMContext, _file_transferred_to, _get_source +from murfey.client.context import _file_transferred_to, _get_source +from murfey.client.contexts.sim import SIMContext visit_name = "cm12345-6" instrument_name = "sim" @@ -97,24 +98,23 @@ def test_sim_context_initialises(tmp_path: Path): @pytest.mark.parametrize( "test_params", - ( # Has environment | Has source | Has destination + ( # Has environment | Has source # Success case - (True, True, True), + (True, True), # Fail cases - (True, True, False), # No destination - (True, False, True), # No source - (False, True, True), # No environment + (True, False), # No source + (False, True), # No environment ), ) def test_post_transfer( mocker: MockerFixture, - test_params: tuple[bool, bool, bool], + test_params: tuple[bool, bool], tmp_path: Path, visit_dir: Path, sim_data: list[Path], ): # Unpack test params - use_env, has_src, has_dst = test_params + use_env, has_src = test_params # Mock the environment mock_environment = None @@ -140,12 +140,8 @@ def test_post_transfer( mock_get_source.return_value = tmp_path if has_src else None mock_file_transferred_to = mocker.patch( - "murfey.client.contexts.sim._file_transferred_to" + "murfey.client.contexts.sim._file_transferred_to", side_effect=destination_files ) - if has_dst: - mock_file_transferred_to.side_effect = destination_files - else: - mock_file_transferred_to.return_value = None mock_capture_post = mocker.patch("murfey.client.contexts.sim.capture_post") @@ -164,10 +160,6 @@ def test_post_transfer( mock_logger.warning.assert_called_with("No environment passed in") elif not has_src: mock_logger.warning.assert_called_with(f"No source found for file {file}") - elif not has_dst: - mock_logger.warning.assert_called_with( - f"Could not find destination file path for {file.name!r}" - ) else: for src, dst in zip(sim_data, [Path(""), *destination_files]): if src.stem.endswith("_BF"): From 41d4dc7d19d7b19343d32961f4cd5ba43275402f Mon Sep 17 00:00:00 2001 From: Eu Pin Tien Date: Thu, 16 Jul 2026 03:51:45 +0100 Subject: [PATCH 17/17] Print formatted dict instead --- src/murfey/server/api/workflow_sim.py | 6 +++++- tests/server/api/test_workflow_sim.py | 4 +++- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/src/murfey/server/api/workflow_sim.py b/src/murfey/server/api/workflow_sim.py index 83e9e2720..8bc91bc07 100644 --- a/src/murfey/server/api/workflow_sim.py +++ b/src/murfey/server/api/workflow_sim.py @@ -1,3 +1,4 @@ +import json import logging from pathlib import Path @@ -40,7 +41,10 @@ def request_sim_processing(session_id: int, sim_data: SIMDataFile): "feedback_queue": _transport_object.feedback_queue, }, } - logger.debug(f"Will submit the following message to 'processing_recipe':\n{recipe}") + logger.debug( + "Will submit the following message to 'processing_recipe':\n" + f"{json.dumps(recipe, indent=2, default=str)}" + ) # Disabled for now; will submit message once recipe and service have been set up # _transport_object.send( # queue="processing_recipe", message=recipe, new_connection=True diff --git a/tests/server/api/test_workflow_sim.py b/tests/server/api/test_workflow_sim.py index 2da84efd5..a17d7fd92 100644 --- a/tests/server/api/test_workflow_sim.py +++ b/tests/server/api/test_workflow_sim.py @@ -1,3 +1,4 @@ +import json from pathlib import Path from unittest.mock import MagicMock @@ -49,7 +50,8 @@ def test_request_sim_processing( }, } mock_logger.debug.assert_called_with( - f"Will submit the following message to 'processing_recipe':\n{recipe}" + "Will submit the following message to 'processing_recipe':\n" + f"{json.dumps(recipe, indent=2, default=str)}" ) # mock_transport_object.send.assert_called_with( # queue="processing_recipe", message=recipe, new_connection=True