diff --git a/src/hyperion/experiment_plans/robot_load_then_centre_plan.py b/src/hyperion/experiment_plans/robot_load_then_centre_plan.py index 15d0f4ec5..b83151fae 100644 --- a/src/hyperion/experiment_plans/robot_load_then_centre_plan.py +++ b/src/hyperion/experiment_plans/robot_load_then_centre_plan.py @@ -200,6 +200,7 @@ def robot_load_then_centre( yield from read_energy(cast(SetEnergyComposite, composite)) ) + # XXX TODO 1173 remove this - left in to avoid breaking nexus files? parameters.hyperion_params.ispyb_params.current_energy_ev = actual_energy_ev if not parameters.experiment_params.requested_energy_kev: parameters.hyperion_params.detector_params.expected_energy_ev = actual_energy_ev diff --git a/src/hyperion/external_interaction/callbacks/common/ispyb_mapping.py b/src/hyperion/external_interaction/callbacks/common/ispyb_mapping.py index b6ddd2a7c..43b3835d0 100644 --- a/src/hyperion/external_interaction/callbacks/common/ispyb_mapping.py +++ b/src/hyperion/external_interaction/callbacks/common/ispyb_mapping.py @@ -24,12 +24,17 @@ from hyperion.log import ISPYB_LOGGER -def populate_data_collection_group(experiment_type, detector_params, ispyb_params): +def populate_data_collection_group( + experiment_type: str, + detector_params: DetectorParams, + ispyb_params: IspybParams, + sample_barcode: Optional[str] = None, +): dcg_info = DataCollectionGroupInfo( visit_string=get_visit_string(ispyb_params, detector_params), experiment_type=experiment_type, sample_id=ispyb_params.sample_id, - sample_barcode=ispyb_params.sample_barcode, + sample_barcode=sample_barcode, ) return dcg_info @@ -58,12 +63,8 @@ def populate_remaining_data_collection_info( data_collection_info.axis_start = data_collection_info.omega_start data_collection_info.focal_spot_size_at_samplex = ispyb_params.focal_spot_size_x data_collection_info.focal_spot_size_at_sampley = ispyb_params.focal_spot_size_y - data_collection_info.slitgap_vertical = ispyb_params.slit_gap_size_y - data_collection_info.slitgap_horizontal = ispyb_params.slit_gap_size_x data_collection_info.beamsize_at_samplex = ispyb_params.beam_size_x data_collection_info.beamsize_at_sampley = ispyb_params.beam_size_y - # Ispyb wants the transmission in a percentage, we use fractions - data_collection_info.transmission = ispyb_params.transmission_fraction * 100 data_collection_info.comments = comment_constructor() data_collection_info.detector_distance = detector_params.detector_distance data_collection_info.exp_time = detector_params.exposure_time @@ -74,17 +75,13 @@ def populate_remaining_data_collection_info( # planned to be removed later data_collection_info.n_passes = 1 data_collection_info.overlap = 0 - data_collection_info.flux = ispyb_params.flux data_collection_info.start_image_number = 1 data_collection_info.resolution = ispyb_params.resolution - data_collection_info.wavelength = ispyb_params.wavelength_angstroms beam_position = detector_params.get_beam_position_mm( detector_params.detector_distance ) data_collection_info.xbeam = beam_position[0] data_collection_info.ybeam = beam_position[1] - data_collection_info.synchrotron_mode = ispyb_params.synchrotron_mode - data_collection_info.undulator_gap1 = ispyb_params.undulator_gap data_collection_info.start_time = get_current_time_string() # temporary file template until nxs filewriting is integrated and we can use # that file name diff --git a/src/hyperion/external_interaction/callbacks/ispyb_callback_base.py b/src/hyperion/external_interaction/callbacks/ispyb_callback_base.py index 5f04681e3..ed45d78c8 100644 --- a/src/hyperion/external_interaction/callbacks/ispyb_callback_base.py +++ b/src/hyperion/external_interaction/callbacks/ispyb_callback_base.py @@ -1,11 +1,19 @@ from __future__ import annotations from abc import abstractmethod +from collections.abc import Sequence from typing import TYPE_CHECKING, Any, Callable, Dict, Optional, TypeVar +from hyperion.external_interaction.callbacks.common.ispyb_mapping import ( + populate_data_collection_group, +) from hyperion.external_interaction.callbacks.plan_reactive_callback import ( PlanReactiveCallback, ) +from hyperion.external_interaction.ispyb.data_model import ( + DataCollectionInfo, + ScanDataInfo, +) from hyperion.external_interaction.ispyb.ispyb_store import ( IspybIds, StoreInIspyb, @@ -13,12 +21,14 @@ from hyperion.external_interaction.ispyb.ispyb_utils import get_ispyb_config from hyperion.log import ISPYB_LOGGER, set_dcgid_tag from hyperion.parameters.constants import CONST +from hyperion.parameters.internal_parameters import InternalParameters from hyperion.parameters.plan_specific.gridscan_internal_params import ( GridscanInternalParameters, ) from hyperion.parameters.plan_specific.rotation_scan_internal_params import ( RotationInternalParameters, ) +from hyperion.utils.utils import convert_eV_to_angstrom D = TypeVar("D") if TYPE_CHECKING: @@ -36,6 +46,8 @@ def __init__( for self.ispyb_ids.""" ISPYB_LOGGER.debug("Initialising ISPyB callback") super().__init__(log=ISPYB_LOGGER, emit=emit) + self._event_driven_data_collection_info: Optional[DataCollectionInfo] = None + self._sample_barcode: Optional[str] = None self.params: GridscanInternalParameters | RotationInternalParameters | None = ( None ) @@ -56,6 +68,8 @@ def __init__( self.log = ISPYB_LOGGER def activity_gated_start(self, doc: RunStart): + self._event_driven_data_collection_info = DataCollectionInfo() + self._sample_barcode = None return self._tag_doc(doc) def activity_gated_descriptor(self, doc: EventDescriptor): @@ -77,41 +91,83 @@ def activity_gated_event(self, doc: Event) -> Event: ) return doc if event_descriptor.get("name") == CONST.PLAN.ISPYB_HARDWARE_READ: + assert self._event_driven_data_collection_info ISPYB_LOGGER.info("ISPyB handler received event from read hardware") - self.params.hyperion_params.ispyb_params.undulator_gap = doc["data"][ + self._event_driven_data_collection_info.undulator_gap1 = doc["data"][ "undulator_current_gap" ] - self.params.hyperion_params.ispyb_params.synchrotron_mode = doc["data"][ + self._event_driven_data_collection_info.synchrotron_mode = doc["data"][ "synchrotron-synchrotron_mode" ] - self.params.hyperion_params.ispyb_params.slit_gap_size_x = doc["data"][ + self._event_driven_data_collection_info.slitgap_horizontal = doc["data"][ "s4_slit_gaps_xgap" ] - self.params.hyperion_params.ispyb_params.slit_gap_size_y = doc["data"][ + self._event_driven_data_collection_info.slitgap_vertical = doc["data"][ "s4_slit_gaps_ygap" ] - self.params.hyperion_params.ispyb_params.sample_barcode = doc["data"][ - "robot-barcode" - ] + self._sample_barcode = doc["data"]["robot-barcode"] if event_descriptor.get("name") == CONST.PLAN.ISPYB_TRANSMISSION_FLUX_READ: - self.params.hyperion_params.ispyb_params.transmission_fraction = doc[ - "data" - ]["attenuator_actual_transmission"] - self.params.hyperion_params.ispyb_params.flux = doc["data"][ + assert self._event_driven_data_collection_info + if doc["data"]["attenuator_actual_transmission"]: + # Ispyb wants the transmission in a percentage, we use fractions + self._event_driven_data_collection_info.transmission = ( + doc["data"]["attenuator_actual_transmission"] * 100 + ) + # TODO 1173 Remove this once nexus_utils no longer needs it + self.params.hyperion_params.ispyb_params.transmission_fraction = doc[ + "data" + ]["attenuator_actual_transmission"] + self._event_driven_data_collection_info.flux = doc["data"][ "flux_flux_reading" ] - self.params.hyperion_params.ispyb_params.current_energy_ev = ( - doc["data"]["dcm_energy_in_kev"] * 1000 + # TODO 1173 Remove this once nexus_utils no longer needs it + self.params.hyperion_params.ispyb_params.flux = ( + self._event_driven_data_collection_info.flux + ) + if doc["data"]["dcm_energy_in_kev"]: + energy_ev = doc["data"]["dcm_energy_in_kev"] * 1000 + self._event_driven_data_collection_info.wavelength = ( + convert_eV_to_angstrom(energy_ev) + ) + # TODO 1173 Remove this once nexus_utils no longer needs wavelength_angstroms + self.params.hyperion_params.ispyb_params.current_energy_ev = energy_ev + + scan_data_infos = self.populate_info_for_update( + self._event_driven_data_collection_info, self.params ) - ISPYB_LOGGER.info("Updating ispyb entry.") - self.ispyb_ids = self.update_deposition(self.params) + self.ispyb_ids = self.update_deposition( + self.params, + scan_data_infos, + self._sample_barcode, + ) ISPYB_LOGGER.info(f"Recieved ISPYB IDs: {self.ispyb_ids}") return self._tag_doc(doc) + def update_deposition( + self, + params, + scan_data_infos: Sequence[ScanDataInfo], + sample_barcode: Optional[str], + ) -> IspybIds: + data_collection_group_info = populate_data_collection_group( + self.ispyb.experiment_type, + params.hyperion_params.detector_params, + params.hyperion_params.ispyb_params, + sample_barcode, + ) + + return self.ispyb.update_deposition( + self.ispyb_ids, data_collection_group_info, scan_data_infos + ) + @abstractmethod - def update_deposition(self, params) -> IspybIds: + def populate_info_for_update( + self, + event_sourced_data_collection_info: DataCollectionInfo, + params: InternalParameters, + ) -> Sequence[ScanDataInfo]: pass def activity_gated_stop(self, doc: RunStop) -> RunStop: diff --git a/src/hyperion/external_interaction/callbacks/rotation/ispyb_callback.py b/src/hyperion/external_interaction/callbacks/rotation/ispyb_callback.py index 17dfdc189..4988fc9c7 100644 --- a/src/hyperion/external_interaction/callbacks/rotation/ispyb_callback.py +++ b/src/hyperion/external_interaction/callbacks/rotation/ispyb_callback.py @@ -1,6 +1,8 @@ from __future__ import annotations -from typing import TYPE_CHECKING, Any, Callable +from collections.abc import Sequence +from dataclasses import asdict, replace +from typing import TYPE_CHECKING, Any, Callable, cast from hyperion.external_interaction.callbacks.common.ispyb_mapping import ( populate_data_collection_group, @@ -14,7 +16,11 @@ construct_comment_for_rotation_scan, populate_data_collection_info_for_rotation, ) -from hyperion.external_interaction.ispyb.data_model import ExperimentType, ScanDataInfo +from hyperion.external_interaction.ispyb.data_model import ( + DataCollectionInfo, + ExperimentType, + ScanDataInfo, +) from hyperion.external_interaction.ispyb.ispyb_store import ( IspybIds, StoreInIspyb, @@ -120,29 +126,37 @@ def activity_gated_start(self, doc: RunStart): self.uid_to_finalize_on = doc.get("uid") return super().activity_gated_start(doc) - def update_deposition(self, params): - dcg_info = populate_data_collection_group( - self.ispyb.experiment_type, - params.hyperion_params.detector_params, + def populate_info_for_update( + self, event_sourced_data_collection_info: DataCollectionInfo, params + ) -> Sequence[ScanDataInfo]: + params = cast(RotationInternalParameters, params) + initial_collection_info = populate_data_collection_info_for_rotation( params.hyperion_params.ispyb_params, + params.hyperion_params.detector_params, + params, ) - scan_data_info = ScanDataInfo( - data_collection_info=populate_remaining_data_collection_info( - construct_comment_for_rotation_scan, - self.ispyb_ids.data_collection_group_id, - populate_data_collection_info_for_rotation( - params.hyperion_params.ispyb_params, + initial_collection_info = replace( + initial_collection_info, + **{ + k: v + for (k, v) in asdict(event_sourced_data_collection_info).items() + if v + }, + ) + return [ + ScanDataInfo( + data_collection_info=populate_remaining_data_collection_info( + construct_comment_for_rotation_scan, + self.ispyb_ids.data_collection_group_id, + initial_collection_info, params.hyperion_params.detector_params, - params, + params.hyperion_params.ispyb_params, ), - params.hyperion_params.detector_params, - params.hyperion_params.ispyb_params, - ), - data_collection_position_info=populate_data_collection_position_info( - params.hyperion_params.ispyb_params - ), - ) - return self.ispyb.update_deposition(self.ispyb_ids, dcg_info, [scan_data_info]) + data_collection_position_info=populate_data_collection_position_info( + params.hyperion_params.ispyb_params + ), + ) + ] def activity_gated_event(self, doc: Event): doc = super().activity_gated_event(doc) diff --git a/src/hyperion/external_interaction/callbacks/xray_centre/ispyb_callback.py b/src/hyperion/external_interaction/callbacks/xray_centre/ispyb_callback.py index 906a572ee..764e7830f 100644 --- a/src/hyperion/external_interaction/callbacks/xray_centre/ispyb_callback.py +++ b/src/hyperion/external_interaction/callbacks/xray_centre/ispyb_callback.py @@ -1,7 +1,9 @@ from __future__ import annotations +from collections.abc import Sequence +from dataclasses import asdict, replace from time import time -from typing import TYPE_CHECKING, Any, Callable, List, Optional +from typing import TYPE_CHECKING, Any, Callable, List, Optional, cast import numpy as np from dodal.devices.zocalo.zocalo_results import ZOCALO_READING_PLAN_NAME @@ -22,7 +24,11 @@ populate_xz_data_collection_info, ) from hyperion.external_interaction.exceptions import ISPyBDepositionNotMade -from hyperion.external_interaction.ispyb.data_model import ExperimentType, ScanDataInfo +from hyperion.external_interaction.ispyb.data_model import ( + DataCollectionInfo, + ExperimentType, + ScanDataInfo, +) from hyperion.external_interaction.ispyb.ispyb_store import ( IspybIds, StoreInIspyb, @@ -163,22 +169,24 @@ def activity_gated_event(self, doc: Event): return doc - def update_deposition(self, params): - data_collection_group_info = populate_data_collection_group( - self.ispyb.experiment_type, - params.hyperion_params.detector_params, - params.hyperion_params.ispyb_params, - ) - - scan_data_infos = [self.populate_xy_scan_data_info(params)] + def populate_info_for_update( + self, event_sourced_data_collection_info: DataCollectionInfo, params + ) -> Sequence[ScanDataInfo]: + params = cast(GridscanInternalParameters, params) + scan_data_infos = [ + self.populate_xy_scan_data_info(params, event_sourced_data_collection_info) + ] if self.is_3d_gridscan(): - scan_data_infos.append(self.populate_xz_scan_data_info(params)) - - return self.ispyb.update_deposition( - self.ispyb_ids, data_collection_group_info, scan_data_infos - ) + scan_data_infos.append( + self.populate_xz_scan_data_info( + params, event_sourced_data_collection_info + ) + ) + return scan_data_infos - def populate_xy_scan_data_info(self, params): + def populate_xy_scan_data_info( + self, params, event_sourced_data_collection_info: DataCollectionInfo + ): grid_scan_info = GridScanInfo( [ int(params.hyperion_params.ispyb_params.upper_left[0]), @@ -195,6 +203,15 @@ def populate_xy_scan_data_info(self, params): params.hyperion_params.detector_params, ) + xy_data_collection_info = replace( + xy_data_collection_info, + **{ + k: v + for (k, v) in asdict(event_sourced_data_collection_info).items() + if v + }, + ) + def comment_constructor(): return construct_comment_for_gridscan( params, params.hyperion_params.ispyb_params, grid_scan_info @@ -218,7 +235,9 @@ def comment_constructor(): ), ) - def populate_xz_scan_data_info(self, params): + def populate_xz_scan_data_info( + self, params, event_sourced_data_collection_info: DataCollectionInfo + ): xz_grid_scan_info = GridScanInfo( [ int(params.hyperion_params.ispyb_params.upper_left[0]), @@ -233,6 +252,14 @@ def populate_xz_scan_data_info(self, params): params.hyperion_params.ispyb_params, params.hyperion_params.detector_params, ) + xz_data_collection_info = replace( + xz_data_collection_info, + **{ + k: v + for (k, v) in asdict(event_sourced_data_collection_info).items() + if v + }, + ) def xz_comment_constructor(): return construct_comment_for_gridscan( diff --git a/src/hyperion/external_interaction/ispyb/data_model.py b/src/hyperion/external_interaction/ispyb/data_model.py index 714ae0f72..63b74f247 100644 --- a/src/hyperion/external_interaction/ispyb/data_model.py +++ b/src/hyperion/external_interaction/ispyb/data_model.py @@ -55,7 +55,7 @@ class DataCollectionGroupInfo: visit_string: str experiment_type: str sample_id: Optional[str] - sample_barcode: Optional[str] + sample_barcode: Optional[str] = None @dataclass(kw_only=True) diff --git a/src/hyperion/external_interaction/ispyb/ispyb_dataclass.py b/src/hyperion/external_interaction/ispyb/ispyb_dataclass.py index 5b3de5d63..1efc3011e 100644 --- a/src/hyperion/external_interaction/ispyb/ispyb_dataclass.py +++ b/src/hyperion/external_interaction/ispyb/ispyb_dataclass.py @@ -8,30 +8,24 @@ GRIDSCAN_ISPYB_PARAM_DEFAULTS = { "sample_id": None, - "sample_barcode": None, "visit_path": "", "microns_per_pixel_x": 0.0, "microns_per_pixel_y": 0.0, - "current_energy_ev": 12700, - # populate later depending on if requested energy is specified - "expected_energy_ev": None, + "current_energy_ev": None, # gets stored as 2x2D coords - (x, y) and (x, z). Values in pixels "upper_left": [0, 0, 0], "position": [0, 0, 0], "xtal_snapshots_omega_start": ["test_1_y", "test_2_y", "test_3_y"], "xtal_snapshots_omega_end": ["test_1_z", "test_2_z", "test_3_z"], - "transmission_fraction": 1.0, - "flux": 10.0, + "transmission_fraction": None, + "flux": None, "beam_size_x": 0.1, "beam_size_y": 0.1, "focal_spot_size_x": 0.0, "focal_spot_size_y": 0.0, "comment": "Descriptive comment.", "resolution": 1, - "undulator_gap": 1.0, - "synchrotron_mode": None, - "slit_gap_size_x": 0.1, - "slit_gap_size_y": 0.1, + "undulator_gap": None, } @@ -41,7 +35,7 @@ class IspybParams(BaseModel): microns_per_pixel_y: float position: np.ndarray - transmission_fraction: float + transmission_fraction: Optional[float] # TODO 1033 this is now deprecated # populated by robot_load_then_centre current_energy_ev: Optional[float] beam_size_x: float @@ -53,14 +47,10 @@ class IspybParams(BaseModel): resolution: Optional[float] sample_id: Optional[str] = None - sample_barcode: Optional[str] = None # Optional from GDA as populated by Ophyd flux: Optional[float] = None undulator_gap: Optional[float] = None - synchrotron_mode: Optional[str] = None - slit_gap_size_x: Optional[float] = None - slit_gap_size_y: Optional[float] = None xtal_snapshots_omega_start: Optional[list[str]] = None xtal_snapshots_omega_end: Optional[list[str]] = None @@ -85,8 +75,8 @@ def _parse_position( return np.array(position) @validator("transmission_fraction") - def _transmission_not_percentage(cls, transmission_fraction: float): - if transmission_fraction > 1: + def _transmission_not_percentage(cls, transmission_fraction: Optional[float]): + if transmission_fraction and transmission_fraction > 1: raise ValueError( "Transmission_fraction of >1 given. Did you give a percentage instead of a fraction?" ) diff --git a/src/hyperion/external_interaction/ispyb/ispyb_store.py b/src/hyperion/external_interaction/ispyb/ispyb_store.py index aeb122f9e..64dfca6b0 100755 --- a/src/hyperion/external_interaction/ispyb/ispyb_store.py +++ b/src/hyperion/external_interaction/ispyb/ispyb_store.py @@ -53,42 +53,42 @@ def begin_deposition( data_collection_group_info: DataCollectionGroupInfo, scan_data_info: ScanDataInfo, ) -> IspybIds: - with ispyb.open(self.ISPYB_CONFIG_PATH) as conn: - assert conn is not None - assert ( - scan_data_info.data_collection_info.visit_string - ), "No visit string supplied for ispyb" - data_collection_group_id = scan_data_info.data_collection_info.parent_id - if not data_collection_group_id: - data_collection_group_id = self._store_data_collection_group_table( - conn, data_collection_group_info - ) - scan_data_info.data_collection_info.parent_id = data_collection_group_id - data_collection_id = self._store_data_collection_table( - conn, None, scan_data_info.data_collection_info - ) - return IspybIds( - data_collection_group_id=data_collection_group_id, - data_collection_ids=(data_collection_id,), + ispyb_ids = IspybIds() + if scan_data_info.data_collection_info: + ispyb_ids.data_collection_group_id = ( + scan_data_info.data_collection_info.parent_id ) + return self._begin_or_update_deposition( + ispyb_ids, data_collection_group_info, [scan_data_info] + ) + def update_deposition( self, ispyb_ids, data_collection_group_info: DataCollectionGroupInfo, scan_data_infos: Sequence[ScanDataInfo], + ): + assert ( + ispyb_ids.data_collection_group_id + ), "Attempted to store scan data without a collection group" + assert ( + ispyb_ids.data_collection_ids + ), "Attempted to store scan data without a collection" + return self._begin_or_update_deposition( + ispyb_ids, data_collection_group_info, scan_data_infos + ) + + def _begin_or_update_deposition( + self, ispyb_ids, data_collection_group_info, scan_data_infos ): with ispyb.open(self.ISPYB_CONFIG_PATH) as conn: assert conn is not None, "Failed to connect to ISPyB" - assert ( - ispyb_ids.data_collection_group_id - ), "Attempted to store scan data without a collection group" - assert ( - ispyb_ids.data_collection_ids - ), "Attempted to store scan data without a collection" - - self._store_data_collection_group_table( - conn, data_collection_group_info, ispyb_ids.data_collection_group_id + + ispyb_ids.data_collection_group_id = ( + self._store_data_collection_group_table( + conn, data_collection_group_info, ispyb_ids.data_collection_group_id + ) ) grid_ids = [] @@ -96,6 +96,14 @@ def update_deposition( for scan_data_info, data_collection_id in zip_longest( scan_data_infos, ispyb_ids.data_collection_ids ): + if ( + scan_data_info.data_collection_info + and not scan_data_info.data_collection_info.parent_id + ): + scan_data_info.data_collection_info.parent_id = ( + ispyb_ids.data_collection_group_id + ) + data_collection_id, grid_id = self._store_single_scan_data( conn, scan_data_info, data_collection_id ) @@ -195,18 +203,6 @@ def _store_data_collection_group_table( return self._upsert_data_collection_group(conn, params) - @staticmethod - @TRACER.start_as_current_span("_upsert_data_collection_group") - def _upsert_data_collection_group( - conn: Connector, params: StrictOrderedDict - ) -> int: - return conn.mx_acquisition.upsert_data_collection_group(list(params.values())) - - @staticmethod - @TRACER.start_as_current_span("_upsert_data_collection") - def _upsert_data_collection(conn: Connector, params: StrictOrderedDict) -> int: - return conn.mx_acquisition.upsert_data_collection(list(params.values())) - def _store_data_collection_table( self, conn, data_collection_id, data_collection_info ): @@ -264,3 +260,15 @@ def _fill_common_data_collection_params( } return params + + @staticmethod + @TRACER.start_as_current_span("_upsert_data_collection_group") + def _upsert_data_collection_group( + conn: Connector, params: StrictOrderedDict + ) -> int: + return conn.mx_acquisition.upsert_data_collection_group(list(params.values())) + + @staticmethod + @TRACER.start_as_current_span("_upsert_data_collection") + def _upsert_data_collection(conn: Connector, params: StrictOrderedDict) -> int: + return conn.mx_acquisition.upsert_data_collection(list(params.values())) diff --git a/src/hyperion/external_interaction/nexus/nexus_utils.py b/src/hyperion/external_interaction/nexus/nexus_utils.py index fe7128425..c07b2663f 100644 --- a/src/hyperion/external_interaction/nexus/nexus_utils.py +++ b/src/hyperion/external_interaction/nexus/nexus_utils.py @@ -133,6 +133,6 @@ def create_beam_and_attenuator_parameters( tuple[Beam, Attenuator]: Descriptions of the beam and attenuator for nexgen. """ return ( - Beam(convert_eV_to_angstrom(energy_kev * 1000), flux), - Attenuator(transmission_fraction), + Beam(convert_eV_to_angstrom(energy_kev * 1000), flux), # pyright: ignore + Attenuator(transmission_fraction), # pyright: ignore ) diff --git a/src/hyperion/parameters/schemas/ispyb_parameters_schema.json b/src/hyperion/parameters/schemas/ispyb_parameters_schema.json index 624d6d1a4..8d27ffc20 100644 --- a/src/hyperion/parameters/schemas/ispyb_parameters_schema.json +++ b/src/hyperion/parameters/schemas/ispyb_parameters_schema.json @@ -46,7 +46,7 @@ } }, "flux": { - "type": "number" + "type": ["number", "null"] }, "beam_size_x": { "type": "number" @@ -54,12 +54,6 @@ "beam_size_y": { "type": "number" }, - "slit_gap_size_x": { - "type": "number" - }, - "slit_gap_size_y": { - "type": "number" - }, "focal_spot_size_x": { "type": "number" }, @@ -84,7 +78,6 @@ "microns_per_pixel_x", "microns_per_pixel_y", "position", - "flux", "beam_size_x", "beam_size_y", "focal_spot_size_x", diff --git a/tests/system_tests/external_interaction/callbacks/test_external_callbacks.py b/tests/system_tests/external_interaction/callbacks/test_external_callbacks.py index 22a87b7f2..740f9400c 100644 --- a/tests/system_tests/external_interaction/callbacks/test_external_callbacks.py +++ b/tests/system_tests/external_interaction/callbacks/test_external_callbacks.py @@ -220,6 +220,7 @@ def test_remote_callbacks_write_to_dev_ispyb_for_rotation( test_rotation_params.hyperion_params.ispyb_params.beam_size_x = test_bs_x test_rotation_params.hyperion_params.ispyb_params.beam_size_y = test_bs_y test_rotation_params.hyperion_params.detector_params.exposure_time = test_exp_time + # TODO 1173 this should not be set here test_rotation_params.hyperion_params.ispyb_params.current_energy_ev = ( convert_angstrom_to_eV(test_wl) ) diff --git a/tests/system_tests/external_interaction/conftest.py b/tests/system_tests/external_interaction/conftest.py index 931df4a02..5e11f4d45 100644 --- a/tests/system_tests/external_interaction/conftest.py +++ b/tests/system_tests/external_interaction/conftest.py @@ -6,7 +6,7 @@ import ispyb.sqlalchemy import numpy as np import pytest -from ispyb.sqlalchemy import DataCollection +from ispyb.sqlalchemy import DataCollection, DataCollectionGroup from sqlalchemy import create_engine from sqlalchemy.orm import sessionmaker @@ -83,20 +83,37 @@ def get_current_datacollection_attribute( return data +def get_current_datacollectiongroup_attribute( + Session: Callable, dcg_id: int, attr: str +): + with Session() as session: + query = session.query(DataCollectionGroup).filter( + DataCollection.dataCollectionGroupId == dcg_id + ) + first_result = query.first() + return getattr(first_result, attr) + + @pytest.fixture -def fetch_comment() -> Callable: +def sqlalchemy_sessionmaker() -> sessionmaker: url = ispyb.sqlalchemy.url(CONST.SIM.DEV_ISPYB_DATABASE_CFG) engine = create_engine(url, connect_args={"use_pure": True}) - Session = sessionmaker(engine) - return partial(get_current_datacollection_comment, Session) + return sessionmaker(engine) @pytest.fixture -def fetch_datacollection_attribute() -> Callable: - url = ispyb.sqlalchemy.url(CONST.SIM.DEV_ISPYB_DATABASE_CFG) - engine = create_engine(url, connect_args={"use_pure": True}) - Session = sessionmaker(engine) - return partial(get_current_datacollection_attribute, Session) +def fetch_comment(sqlalchemy_sessionmaker) -> Callable: + return partial(get_current_datacollection_comment, sqlalchemy_sessionmaker) + + +@pytest.fixture +def fetch_datacollection_attribute(sqlalchemy_sessionmaker) -> Callable: + return partial(get_current_datacollection_attribute, sqlalchemy_sessionmaker) + + +@pytest.fixture +def fetch_datacollectiongroup_attribute(sqlalchemy_sessionmaker) -> Callable: + return partial(get_current_datacollectiongroup_attribute, sqlalchemy_sessionmaker) @pytest.fixture diff --git a/tests/system_tests/external_interaction/test_ispyb_dev_connection.py b/tests/system_tests/external_interaction/test_ispyb_dev_connection.py index 7effdd9c0..ce010900a 100644 --- a/tests/system_tests/external_interaction/test_ispyb_dev_connection.py +++ b/tests/system_tests/external_interaction/test_ispyb_dev_connection.py @@ -10,7 +10,7 @@ from dodal.devices.attenuator import Attenuator from dodal.devices.flux import Flux from dodal.devices.s4_slit_gaps import S4SlitGaps -from dodal.devices.synchrotron import Synchrotron +from dodal.devices.synchrotron import Synchrotron, SynchrotronMode from dodal.devices.undulator import Undulator from hyperion.experiment_plans.rotation_scan_plan import ( @@ -311,6 +311,7 @@ def test_ispyb_deposition_in_rotation_plan( test_rotation_params: RotationInternalParameters, fetch_comment: Callable[..., Any], fetch_datacollection_attribute: Callable[..., Any], + fetch_datacollectiongroup_attribute: Callable[..., Any], undulator: Undulator, attenuator: Attenuator, synchrotron: Synchrotron, @@ -324,17 +325,33 @@ def test_ispyb_deposition_in_rotation_plan( test_bs_y = 0.047 test_exp_time = 0.023 test_img_wid = 0.27 + test_undulator_current_gap = 1.12 + test_synchrotron_mode = SynchrotronMode.USER + test_slit_gap_horiz = 0.123 + test_slit_gap_vert = 0.234 test_rotation_params.experiment_params.image_width = test_img_wid test_rotation_params.hyperion_params.ispyb_params.beam_size_x = test_bs_x test_rotation_params.hyperion_params.ispyb_params.beam_size_y = test_bs_y test_rotation_params.hyperion_params.detector_params.exposure_time = test_exp_time - test_rotation_params.hyperion_params.ispyb_params.current_energy_ev = ( - convert_angstrom_to_eV(test_wl) + energy_ev = convert_angstrom_to_eV(test_wl) + fake_create_rotation_devices.dcm.energy_in_kev.user_readback.sim_put( # pyright: ignore + energy_ev / 1000 ) - test_rotation_params.hyperion_params.detector_params.expected_energy_ev = ( - convert_angstrom_to_eV(test_wl) + fake_create_rotation_devices.undulator.current_gap.sim_put(1.12) + fake_create_rotation_devices.synchrotron.machine_status.synchrotron_mode.sim_put( # pyright: ignore + test_synchrotron_mode.value ) + fake_create_rotation_devices.synchrotron.top_up.start_countdown.sim_put( # pyright: ignore + -1 + ) + fake_create_rotation_devices.s4_slit_gaps.xgap.user_readback.sim_put( # pyright: ignore + test_slit_gap_horiz + ) + fake_create_rotation_devices.s4_slit_gaps.ygap.user_readback.sim_put( # pyright: ignore + test_slit_gap_vert + ) + test_rotation_params.hyperion_params.detector_params.expected_energy_ev = energy_ev os.environ["ISPYB_CONFIG_PATH"] = CONST.SIM.DEV_ISPYB_DATABASE_CFG ispyb_cb = RotationISPyBCallback() @@ -366,14 +383,22 @@ def test_ispyb_deposition_in_rotation_plan( dcid = ispyb_cb.ispyb_ids.data_collection_ids[0] assert dcid is not None - comment = fetch_comment(dcid) - assert comment == "Hyperion rotation scan" - wavelength = fetch_datacollection_attribute(dcid, "wavelength") - beamsize_x = fetch_datacollection_attribute(dcid, "beamSizeAtSampleX") - beamsize_y = fetch_datacollection_attribute(dcid, "beamSizeAtSampleY") - exposure = fetch_datacollection_attribute(dcid, "exposureTime") - - assert wavelength == test_wl - assert beamsize_x == test_bs_x - assert beamsize_y == test_bs_y - assert exposure == test_exp_time + assert fetch_comment(dcid) == "Hyperion rotation scan" + assert fetch_datacollection_attribute(dcid, "wavelength") == test_wl + assert fetch_datacollection_attribute(dcid, "beamSizeAtSampleX") == test_bs_x + assert fetch_datacollection_attribute(dcid, "beamSizeAtSampleY") == test_bs_y + assert fetch_datacollection_attribute(dcid, "exposureTime") == test_exp_time + assert ( + fetch_datacollection_attribute(dcid, "undulatorGap1") + == test_undulator_current_gap + ) + assert ( + fetch_datacollection_attribute(dcid, "synchrotronMode") + == test_synchrotron_mode.value + ) + assert ( + fetch_datacollection_attribute(dcid, "slitGapHorizontal") == test_slit_gap_horiz + ) + assert fetch_datacollection_attribute(dcid, "slitGapVertical") == test_slit_gap_vert + # TODO Can't test barcode as need BLSample which needs Dewar, Shipping, Container entries for the + # upsert stored proc to use it. diff --git a/tests/test_data/hyperion_parameters.json b/tests/test_data/hyperion_parameters.json index e497198fc..4ebffa3c0 100644 --- a/tests/test_data/hyperion_parameters.json +++ b/tests/test_data/hyperion_parameters.json @@ -36,7 +36,6 @@ ], "current_energy_ev": 100.0, "transmission_fraction": 1.0, - "flux": 10.0, "beam_size_x": 1.0, "beam_size_y": 1.0, "focal_spot_size_x": 1.0, @@ -44,11 +43,6 @@ "comment": "test", "resolution": 1.0, "sample_id": null, - "sample_barcode": null, - "undulator_gap": null, - "synchrotron_mode": null, - "slit_gap_size_x": 1.0, - "slit_gap_size_y": 1.0, "xtal_snapshots_omega_start": [ "test_1_y", "test_2_y", diff --git a/tests/test_data/parameter_json_files/bad_test_parameters_wrong_version.json b/tests/test_data/parameter_json_files/bad_test_parameters_wrong_version.json index 563567fc2..c88c30f17 100644 --- a/tests/test_data/parameter_json_files/bad_test_parameters_wrong_version.json +++ b/tests/test_data/parameter_json_files/bad_test_parameters_wrong_version.json @@ -7,7 +7,7 @@ "zocalo_environment": "dev_artemis", "experiment_type": "grid_scan", "detector_params": { - "current_energy_ev": 100, + "expected_energy_ev": 100, "exposure_time": 0.1, "directory": "/tmp", "prefix": "file_name", @@ -48,11 +48,8 @@ "test_3" ], "transmission_fraction": 1.0, - "flux": 10.0, "beam_size_x": 1.0, "beam_size_y": 1.0, - "slit_gap_size_x": 1.0, - "slit_gap_size_y": 1.0, "focal_spot_size_x": 1.0, "focal_spot_size_y": 1.0, "comment": "test", diff --git a/tests/test_data/parameter_json_files/good_test_grid_with_edge_detect_parameters.json b/tests/test_data/parameter_json_files/good_test_grid_with_edge_detect_parameters.json index 49d3f428b..167b83dc0 100644 --- a/tests/test_data/parameter_json_files/good_test_grid_with_edge_detect_parameters.json +++ b/tests/test_data/parameter_json_files/good_test_grid_with_edge_detect_parameters.json @@ -7,7 +7,7 @@ "zocalo_environment": "dev_artemis", "experiment_type": "full_grid_scan", "detector_params": { - "current_energy_ev": 100, + "expected_energy_ev": 100, "directory": "/tmp", "prefix": "file_name", "run_number": 0, @@ -23,16 +23,12 @@ 20.0, 30.0 ], - "flux": 10.0, "beam_size_x": 1.0, "beam_size_y": 1.0, - "slit_gap_size_x": 1.0, - "slit_gap_size_y": 1.0, "focal_spot_size_x": 1.0, "focal_spot_size_y": 1.0, "comment": "test", - "resolution": 1.0, - "sample_barcode": "test" + "resolution": 1.0 } }, "experiment_params": { diff --git a/tests/test_data/parameter_json_files/good_test_parameters.json b/tests/test_data/parameter_json_files/good_test_parameters.json index 43eac4b89..1aa43dd2b 100644 --- a/tests/test_data/parameter_json_files/good_test_parameters.json +++ b/tests/test_data/parameter_json_files/good_test_parameters.json @@ -7,7 +7,7 @@ "zocalo_environment": "dev_artemis", "experiment_type": "flyscan_xray_centre", "detector_params": { - "current_energy_ev": 100, + "expected_energy_ev": 100, "directory": "/tmp", "prefix": "file_name", "run_number": 0, @@ -43,11 +43,8 @@ "test_2", "test_3" ], - "flux": 10.0, "beam_size_x": 1.0, "beam_size_y": 1.0, - "slit_gap_size_x": 1.0, - "slit_gap_size_y": 1.0, "focal_spot_size_x": 1.0, "focal_spot_size_y": 1.0, "comment": "test", diff --git a/tests/test_data/parameter_json_files/good_test_pin_centre_then_xray_centre_parameters.json b/tests/test_data/parameter_json_files/good_test_pin_centre_then_xray_centre_parameters.json index ba7093ff1..86275fba4 100644 --- a/tests/test_data/parameter_json_files/good_test_pin_centre_then_xray_centre_parameters.json +++ b/tests/test_data/parameter_json_files/good_test_pin_centre_then_xray_centre_parameters.json @@ -7,7 +7,7 @@ "zocalo_environment": "devrmq", "experiment_type": "pin_tip_centre_then_xray_centre", "detector_params": { - "current_energy_ev": 100, + "expected_energy_ev": 100, "directory": "/tmp", "prefix": "file_name", "run_number": 0, @@ -23,16 +23,12 @@ 20.0, 30.0 ], - "flux": 10.0, "beam_size_x": 1.0, "beam_size_y": 1.0, - "slit_gap_size_x": 1.0, - "slit_gap_size_y": 1.0, "focal_spot_size_x": 1.0, "focal_spot_size_y": 1.0, "comment": "test", - "resolution": 1.0, - "sample_barcode": "test" + "resolution": 1.0 } }, "experiment_params": { diff --git a/tests/test_data/parameter_json_files/good_test_robot_load_params.json b/tests/test_data/parameter_json_files/good_test_robot_load_params.json index 863429889..d72f66990 100644 --- a/tests/test_data/parameter_json_files/good_test_robot_load_params.json +++ b/tests/test_data/parameter_json_files/good_test_robot_load_params.json @@ -21,7 +21,6 @@ 0, 0 ], - "flux": 10.0, "beam_size_x": 0.1, "beam_size_y": 0.1, "focal_spot_size_x": 0.0, diff --git a/tests/test_data/parameter_json_files/good_test_robot_load_params_no_energy.json b/tests/test_data/parameter_json_files/good_test_robot_load_params_no_energy.json index 87dbad332..7812a1c69 100644 --- a/tests/test_data/parameter_json_files/good_test_robot_load_params_no_energy.json +++ b/tests/test_data/parameter_json_files/good_test_robot_load_params_no_energy.json @@ -21,15 +21,13 @@ 0, 0 ], - "flux": 10.0, "beam_size_x": 0.1, "beam_size_y": 0.1, "focal_spot_size_x": 0.0, "focal_spot_size_y": 0.0, "comment": "Descriptive comment.", "resolution": 1, - "sample_id": null, - "sample_barcode": null + "sample_id": null } }, "experiment_params": { diff --git a/tests/test_data/parameter_json_files/good_test_rotation_scan_parameters.json b/tests/test_data/parameter_json_files/good_test_rotation_scan_parameters.json index 20a9ab2df..2fd061783 100644 --- a/tests/test_data/parameter_json_files/good_test_rotation_scan_parameters.json +++ b/tests/test_data/parameter_json_files/good_test_rotation_scan_parameters.json @@ -43,11 +43,8 @@ "test_2", "test_3" ], - "flux": 10.0, "beam_size_x": 1.0, "beam_size_y": 1.0, - "slit_gap_size_x": 1.0, - "slit_gap_size_y": 1.0, "focal_spot_size_x": 1.0, "focal_spot_size_y": 1.0, "comment": "test", diff --git a/tests/test_data/parameter_json_files/good_test_rotation_scan_parameters_nomove.json b/tests/test_data/parameter_json_files/good_test_rotation_scan_parameters_nomove.json index ceb5987a7..c7af6f3c4 100644 --- a/tests/test_data/parameter_json_files/good_test_rotation_scan_parameters_nomove.json +++ b/tests/test_data/parameter_json_files/good_test_rotation_scan_parameters_nomove.json @@ -43,11 +43,8 @@ "test_2", "test_3" ], - "flux": 10.0, "beam_size_x": 1.0, "beam_size_y": 1.0, - "slit_gap_size_x": 1.0, - "slit_gap_size_y": 1.0, "focal_spot_size_x": 1.0, "focal_spot_size_y": 1.0, "comment": "test", diff --git a/tests/test_data/parameter_json_files/good_test_stepped_grid_scan_parameters.json b/tests/test_data/parameter_json_files/good_test_stepped_grid_scan_parameters.json index 25458b062..f20ff1752 100644 --- a/tests/test_data/parameter_json_files/good_test_stepped_grid_scan_parameters.json +++ b/tests/test_data/parameter_json_files/good_test_stepped_grid_scan_parameters.json @@ -7,7 +7,7 @@ "zocalo_environment": "devrmq", "experiment_type": "flyscan_xray_centre", "detector_params": { - "current_energy_ev": 100, + "expected_energy_ev": 100, "directory": "/tmp", "prefix": "file_name", "run_number": 0, @@ -44,11 +44,8 @@ "test_2", "test_3" ], - "flux": 10.0, "beam_size_x": 1.0, "beam_size_y": 1.0, - "slit_gap_size_x": 1.0, - "slit_gap_size_y": 1.0, "focal_spot_size_x": 1.0, "focal_spot_size_y": 1.0, "comment": "test", diff --git a/tests/test_data/parameter_json_files/live_test_rotation_params.json b/tests/test_data/parameter_json_files/live_test_rotation_params.json index 0f84ec59f..bccafacbf 100644 --- a/tests/test_data/parameter_json_files/live_test_rotation_params.json +++ b/tests/test_data/parameter_json_files/live_test_rotation_params.json @@ -7,7 +7,7 @@ "zocalo_environment": "dev_artemis", "experiment_type": "rotation_scan", "detector_params": { - "current_energy_ev": 12700, + "expected_energy_ev": 12700, "directory": "/dls/i03/data/2023/cm33866-3/rotation_scan_test", "prefix": "rotation_scan_test", "run_number": 0, @@ -43,11 +43,8 @@ "test_2", "test_3" ], - "flux": 10.0, "beam_size_x": 1.0, "beam_size_y": 1.0, - "slit_gap_size_x": 1.0, - "slit_gap_size_y": 1.0, "focal_spot_size_x": 1.0, "focal_spot_size_y": 1.0, "comment": "test", diff --git a/tests/test_data/parameter_json_files/live_test_rotation_params_move_xyz.json b/tests/test_data/parameter_json_files/live_test_rotation_params_move_xyz.json index a37300aad..a6aacb97b 100644 --- a/tests/test_data/parameter_json_files/live_test_rotation_params_move_xyz.json +++ b/tests/test_data/parameter_json_files/live_test_rotation_params_move_xyz.json @@ -7,7 +7,7 @@ "zocalo_environment": "dev_artemis", "experiment_type": "rotation_scan", "detector_params": { - "current_energy_ev": 12700, + "expected_energy_ev": 12700, "directory": "/dls/i03/data/2023/cm33866-3/rotation_scan_test", "prefix": "rotation_scan_test", "run_number": 0, @@ -43,11 +43,8 @@ "test_2", "test_3" ], - "flux": 10.0, "beam_size_x": 1.0, "beam_size_y": 1.0, - "slit_gap_size_x": 1.0, - "slit_gap_size_y": 1.0, "focal_spot_size_x": 1.0, "focal_spot_size_y": 1.0, "comment": "test", diff --git a/tests/test_data/parameter_json_files/panda_test_parameters.json b/tests/test_data/parameter_json_files/panda_test_parameters.json index 40fc9f90f..3063b98df 100644 --- a/tests/test_data/parameter_json_files/panda_test_parameters.json +++ b/tests/test_data/parameter_json_files/panda_test_parameters.json @@ -15,7 +15,7 @@ "det_dist_to_beam_converter_path": "tests/test_data/test_lookup_table.txt" }, "ispyb_params": { - "current_energy_ev": 12700, + "expected_energy_ev": 12700, "visit_path": "/dls/i03/data/2023/cm33866-5/", "microns_per_pixel_x": 1.0, "microns_per_pixel_y": 1.0, @@ -44,11 +44,8 @@ "test_2", "test_3" ], - "flux": 10.0, "beam_size_x": 1.0, "beam_size_y": 1.0, - "slit_gap_size_x": 1.0, - "slit_gap_size_y": 1.0, "focal_spot_size_x": 1.0, "focal_spot_size_y": 1.0, "comment": "test", diff --git a/tests/test_data/parameter_json_files/system_test_parameter_defaults.json b/tests/test_data/parameter_json_files/system_test_parameter_defaults.json index 8ecb8e8b2..967b2204f 100644 --- a/tests/test_data/parameter_json_files/system_test_parameter_defaults.json +++ b/tests/test_data/parameter_json_files/system_test_parameter_defaults.json @@ -37,19 +37,13 @@ "test_2_z", "test_3_z" ], - "flux": 10.0, "beam_size_x": 0.1, "beam_size_y": 0.1, "focal_spot_size_x": 0.0, "focal_spot_size_y": 0.0, "comment": "Descriptive comment.", "resolution": 1, - "sample_id": null, - "sample_barcode": null, - "undulator_gap": 1.0, - "synchrotron_mode": null, - "slit_gap_size_x": 0.1, - "slit_gap_size_y": 0.1 + "sample_id": null } }, "experiment_params": { diff --git a/tests/test_data/parameter_json_files/test_internal_parameter_defaults.json b/tests/test_data/parameter_json_files/test_internal_parameter_defaults.json index c8612c505..3a55a6395 100644 --- a/tests/test_data/parameter_json_files/test_internal_parameter_defaults.json +++ b/tests/test_data/parameter_json_files/test_internal_parameter_defaults.json @@ -14,7 +14,7 @@ "det_dist_to_beam_converter_path": "tests/test_data/test_lookup_table.txt" }, "ispyb_params": { - "current_energy_ev": 100, + "expected_energy_ev": 100, "visit_path": "/tmp/cm31105-4", "microns_per_pixel_x": 1.0, "microns_per_pixel_y": 1.0, @@ -38,19 +38,13 @@ "test_2_z", "test_3_z" ], - "flux": 10.0, "beam_size_x": 0.1, "beam_size_y": 0.1, "focal_spot_size_x": 0.0, "focal_spot_size_y": 0.0, "comment": "Descriptive comment.", "resolution": 1, - "sample_id": null, - "sample_barcode": null, - "undulator_gap": 1.0, - "synchrotron_mode": null, - "slit_gap_size_x": 0.1, - "slit_gap_size_y": 0.1 + "sample_id": null } }, "experiment_params": { diff --git a/tests/test_data/parameter_json_files/test_parameter_defaults.json b/tests/test_data/parameter_json_files/test_parameter_defaults.json index 112f079f9..66f35efad 100644 --- a/tests/test_data/parameter_json_files/test_parameter_defaults.json +++ b/tests/test_data/parameter_json_files/test_parameter_defaults.json @@ -37,19 +37,13 @@ "test_2_z", "test_3_z" ], - "flux": 10.0, "beam_size_x": 0.1, "beam_size_y": 0.1, "focal_spot_size_x": 0.0, "focal_spot_size_y": 0.0, "comment": "Descriptive comment.", "resolution": 1, - "sample_id": "0001", - "sample_barcode": "12345A", - "undulator_gap": 1.0, - "synchrotron_mode": null, - "slit_gap_size_x": 0.1, - "slit_gap_size_y": 0.1 + "sample_id": "0001" } }, "experiment_params": { diff --git a/tests/test_data/parameter_json_files/test_parameter_defaults_2d.json b/tests/test_data/parameter_json_files/test_parameter_defaults_2d.json index 82a1be42d..aea4eac11 100644 --- a/tests/test_data/parameter_json_files/test_parameter_defaults_2d.json +++ b/tests/test_data/parameter_json_files/test_parameter_defaults_2d.json @@ -37,19 +37,13 @@ "test_2_z", "test_3_z" ], - "flux": 10.0, "beam_size_x": 0.1, "beam_size_y": 0.1, "focal_spot_size_x": 0.0, "focal_spot_size_y": 0.0, "comment": "Descriptive comment.", "resolution": 1, - "sample_id": "0001", - "sample_barcode": "12345A", - "undulator_gap": 1.0, - "synchrotron_mode": null, - "slit_gap_size_x": 0.1, - "slit_gap_size_y": 0.1 + "sample_id": "0001" } }, "experiment_params": { diff --git a/tests/test_data/parameter_json_files/test_parameters.json b/tests/test_data/parameter_json_files/test_parameters.json index 0a6c71665..a8cc26ed9 100644 --- a/tests/test_data/parameter_json_files/test_parameters.json +++ b/tests/test_data/parameter_json_files/test_parameters.json @@ -7,7 +7,7 @@ "zocalo_environment": "dev_artemis", "experiment_type": "flyscan_xray_centre", "detector_params": { - "current_energy_ev": 100, + "expected_energy_ev": 100, "directory": "/tmp", "prefix": "file_name", "run_number": 0, @@ -43,11 +43,8 @@ "test_2", "test_3" ], - "flux": 10.0, "beam_size_x": 1.0, "beam_size_y": 1.0, - "slit_gap_size_x": 1.0, - "slit_gap_size_y": 1.0, "focal_spot_size_x": 1.0, "focal_spot_size_y": 1.0, "comment": "test", diff --git a/tests/unit_tests/experiment_plans/conftest.py b/tests/unit_tests/experiment_plans/conftest.py index 984739c1c..0ff5d09c6 100644 --- a/tests/unit_tests/experiment_plans/conftest.py +++ b/tests/unit_tests/experiment_plans/conftest.py @@ -43,7 +43,6 @@ def make_event_doc(data, descriptor="abc123") -> Event: BASIC_PRE_SETUP_DOC = { "undulator_current_gap": 0, - "undulator_gap": 0, "synchrotron-synchrotron_mode": 0, "s4_slit_gaps_xgap": 0, "s4_slit_gaps_ygap": 0, @@ -119,24 +118,31 @@ def modified_store_grid_scan_mock(*args, dcids=(0, 0), dcgid=0, **kwargs): @pytest.fixture def mock_subscriptions(test_fgs_params): - with patch( - "hyperion.external_interaction.callbacks.zocalo_callback.ZocaloTrigger", - modified_interactor_mock, - ), patch( - "hyperion.external_interaction.callbacks.xray_centre.ispyb_callback.StoreInIspyb.append_to_comment" - ), patch( - "hyperion.external_interaction.callbacks.xray_centre.ispyb_callback.StoreInIspyb.begin_deposition", - new=MagicMock( - return_value=IspybIds( - data_collection_ids=(0, 0), data_collection_group_id=0 - ) + with ( + patch( + "hyperion.external_interaction.callbacks.zocalo_callback.ZocaloTrigger", + modified_interactor_mock, ), - ), patch( - "hyperion.external_interaction.callbacks.xray_centre.ispyb_callback.StoreInIspyb.update_deposition", - new=MagicMock( - return_value=IspybIds( - data_collection_ids=(0, 0), data_collection_group_id=0, grid_ids=(0, 0) - ) + patch( + "hyperion.external_interaction.callbacks.xray_centre.ispyb_callback.StoreInIspyb.append_to_comment" + ), + patch( + "hyperion.external_interaction.callbacks.xray_centre.ispyb_callback.StoreInIspyb.begin_deposition", + new=MagicMock( + return_value=IspybIds( + data_collection_ids=(0, 0), data_collection_group_id=0 + ) + ), + ), + patch( + "hyperion.external_interaction.callbacks.xray_centre.ispyb_callback.StoreInIspyb.update_deposition", + new=MagicMock( + return_value=IspybIds( + data_collection_ids=(0, 0), + data_collection_group_id=0, + grid_ids=(0, 0), + ) + ), ), ): nexus_callback, ispyb_callback = create_gridscan_callbacks() @@ -173,3 +179,11 @@ def simple_beamline(detector_motion, oav, smargon, synchrotron, test_config_file ) oav.parameters.update_on_zoom(7.5, 1024, 768) return magic_mock + + +def assert_event(mock_call, expected): + actual = mock_call.args[0] + if "data" in actual: + actual = actual["data"] + for k, v in expected.items(): + assert actual[k] == v, f"Mismatch in key {k}, {actual} <=> {expected}" diff --git a/tests/unit_tests/experiment_plans/test_flyscan_xray_centre_plan.py b/tests/unit_tests/experiment_plans/test_flyscan_xray_centre_plan.py index 0bbf01c8a..ee4a6b7f2 100644 --- a/tests/unit_tests/experiment_plans/test_flyscan_xray_centre_plan.py +++ b/tests/unit_tests/experiment_plans/test_flyscan_xray_centre_plan.py @@ -1,7 +1,7 @@ import random import types from typing import Tuple -from unittest.mock import MagicMock, call, patch +from unittest.mock import DEFAULT, MagicMock, call, patch import bluesky.preprocessors as bpp import numpy as np @@ -42,6 +42,9 @@ from hyperion.external_interaction.callbacks.logging_callback import ( VerbosePlanExecutionLoggingCallback, ) +from hyperion.external_interaction.callbacks.plan_reactive_callback import ( + PlanReactiveCallback, +) from hyperion.external_interaction.callbacks.xray_centre.ispyb_callback import ( GridscanISPyBCallback, ) @@ -53,8 +56,8 @@ ) from hyperion.external_interaction.ispyb.ispyb_store import ( IspybIds, - StoreInIspyb, ) +from hyperion.log import ISPYB_LOGGER from hyperion.parameters.constants import CONST from hyperion.parameters.plan_specific.gridscan_internal_params import ( GridscanInternalParameters, @@ -68,6 +71,7 @@ ) from ..external_interaction.callbacks.conftest import TestData from .conftest import ( + assert_event, mock_zocalo_trigger, modified_interactor_mock, modified_store_grid_scan_mock, @@ -205,46 +209,55 @@ def test_read_hardware_for_ispyb_updates_from_ophyd_devices( ) set_sim_value(fake_fgs_composite.robot.barcode.bare_signal, ["BARCODE"]) - test_ispyb_callback = GridscanISPyBCallback() + test_ispyb_callback = PlanReactiveCallback(ISPYB_LOGGER) test_ispyb_callback.active = True - test_ispyb_callback.ispyb = MagicMock(spec=StoreInIspyb) - test_ispyb_callback.ispyb.begin_deposition.return_value = IspybIds( - data_collection_ids=(2, 3), data_collection_group_id=5, grid_ids=(7, 8, 9) - ) - RE.subscribe(test_ispyb_callback) - RE( - ispyb_plan( - fake_fgs_composite.undulator, - fake_fgs_composite.synchrotron, - fake_fgs_composite.s4_slit_gaps, - fake_fgs_composite.robot, - fake_fgs_composite.attenuator, - fake_fgs_composite.flux, - fake_fgs_composite.dcm, - fake_fgs_composite.aperture_scatterguard, - ) - ) - hyperion_params = test_ispyb_callback.params.hyperion_params - - assert hyperion_params.ispyb_params.undulator_gap == undulator_test_value # type: ignore - assert ( - hyperion_params.ispyb_params.synchrotron_mode # type: ignore - == synchrotron_test_value.value - ) - assert hyperion_params.ispyb_params.slit_gap_size_x == xgap_test_value # type: ignore - assert hyperion_params.ispyb_params.slit_gap_size_y == ygap_test_value # type: ignore - assert ( - hyperion_params.ispyb_params.transmission_fraction # type: ignore - == transmission_test_value - ) - assert hyperion_params.ispyb_params.flux == flux_test_value # type: ignore - assert ( - hyperion_params.ispyb_params.current_energy_ev - == current_energy_kev_test_value * 1000 - ) + with patch.multiple( + test_ispyb_callback, + activity_gated_start=DEFAULT, + activity_gated_event=DEFAULT, + ): + RE.subscribe(test_ispyb_callback) - assert hyperion_params.ispyb_params.sample_barcode == "BARCODE" + RE( + ispyb_plan( + fake_fgs_composite.undulator, + fake_fgs_composite.synchrotron, + fake_fgs_composite.s4_slit_gaps, + fake_fgs_composite.robot, + fake_fgs_composite.attenuator, + fake_fgs_composite.flux, + fake_fgs_composite.dcm, + fake_fgs_composite.aperture_scatterguard, + ) + ) + # fmt: off + assert_event( + test_ispyb_callback.activity_gated_start.mock_calls[0], # pyright: ignore + { + "plan_name": "standalone_read_hardware_for_ispyb", + "subplan_name": "run_gridscan_move_and_tidy", + }, + ) + assert_event( + test_ispyb_callback.activity_gated_event.mock_calls[0], # pyright: ignore + { + "undulator_current_gap": undulator_test_value, + "synchrotron-synchrotron_mode": synchrotron_test_value.value, + "s4_slit_gaps_xgap": xgap_test_value, + "s4_slit_gaps_ygap": ygap_test_value, + "robot-barcode": "BARCODE", + }, + ) + assert_event( + test_ispyb_callback.activity_gated_event.mock_calls[1], # pyright: ignore + { + "attenuator_actual_transmission": transmission_test_value, + "flux_flux_reading": flux_test_value, + "dcm_energy_in_kev": current_energy_kev_test_value, + }, + ) + # fmt: on @patch( "dodal.devices.aperturescatterguard.ApertureScatterguard._safe_move_within_datacollection_range", diff --git a/tests/unit_tests/experiment_plans/test_panda_flyscan_xray_centre_plan.py b/tests/unit_tests/experiment_plans/test_panda_flyscan_xray_centre_plan.py index 227c4cf08..398797187 100644 --- a/tests/unit_tests/experiment_plans/test_panda_flyscan_xray_centre_plan.py +++ b/tests/unit_tests/experiment_plans/test_panda_flyscan_xray_centre_plan.py @@ -1,7 +1,7 @@ import random import types from typing import Tuple -from unittest.mock import MagicMock, call, patch +from unittest.mock import DEFAULT, MagicMock, call, patch import bluesky.preprocessors as bpp import numpy as np @@ -39,16 +39,16 @@ from hyperion.external_interaction.callbacks.logging_callback import ( VerbosePlanExecutionLoggingCallback, ) +from hyperion.external_interaction.callbacks.plan_reactive_callback import ( + PlanReactiveCallback, +) from hyperion.external_interaction.callbacks.xray_centre.ispyb_callback import ( GridscanISPyBCallback, ) from hyperion.external_interaction.callbacks.xray_centre.nexus_callback import ( GridscanNexusFileCallback, ) -from hyperion.external_interaction.ispyb.ispyb_store import ( - IspybIds, - StoreInIspyb, -) +from hyperion.log import ISPYB_LOGGER from hyperion.parameters.constants import CONST from hyperion.parameters.plan_specific.panda.panda_gridscan_internal_params import ( PandAGridscanInternalParameters, @@ -61,6 +61,7 @@ ) from ..external_interaction.callbacks.conftest import TestData from .conftest import ( + assert_event, mock_zocalo_trigger, modified_interactor_mock, modified_store_grid_scan_mock, @@ -160,42 +161,53 @@ def test_read_hardware_for_ispyb_updates_from_ophyd_devices( set_sim_value(fake_fgs_composite.robot.barcode.bare_signal, ["BARCODE"]) - test_ispyb_callback = GridscanISPyBCallback() + test_ispyb_callback = PlanReactiveCallback(ISPYB_LOGGER) test_ispyb_callback.active = True - test_ispyb_callback.ispyb = MagicMock(spec=StoreInIspyb) - test_ispyb_callback.ispyb.begin_deposition.return_value = IspybIds( - data_collection_ids=(2, 3), data_collection_group_id=5, grid_ids=(7, 8, 9) - ) - RE.subscribe(test_ispyb_callback) - - RE( - ispyb_plan( - fake_fgs_composite.undulator, - fake_fgs_composite.synchrotron, - fake_fgs_composite.s4_slit_gaps, - fake_fgs_composite.robot, - fake_fgs_composite.attenuator, - fake_fgs_composite.flux, - fake_fgs_composite.dcm, - fake_fgs_composite.aperture_scatterguard, + with patch.multiple( + test_ispyb_callback, + activity_gated_start=DEFAULT, + activity_gated_event=DEFAULT, + ): + RE.subscribe(test_ispyb_callback) + + RE( + ispyb_plan( + fake_fgs_composite.undulator, + fake_fgs_composite.synchrotron, + fake_fgs_composite.s4_slit_gaps, + fake_fgs_composite.robot, + fake_fgs_composite.attenuator, + fake_fgs_composite.flux, + fake_fgs_composite.dcm, + fake_fgs_composite.aperture_scatterguard, + ) ) - ) - params = test_ispyb_callback.params - - assert params.hyperion_params.ispyb_params.undulator_gap == undulator_test_value # type: ignore - assert ( - params.hyperion_params.ispyb_params.synchrotron_mode # type: ignore - == synchrotron_test_value.value - ) - assert params.hyperion_params.ispyb_params.slit_gap_size_x == xgap_test_value # type: ignore - assert params.hyperion_params.ispyb_params.slit_gap_size_y == ygap_test_value # type: ignore - assert ( - params.hyperion_params.ispyb_params.transmission_fraction # type: ignore - == transmission_test_value - ) - assert params.hyperion_params.ispyb_params.flux == flux_test_value # type: ignore - - assert params.hyperion_params.ispyb_params.sample_barcode == "BARCODE" + # fmt: off + assert_event( + test_ispyb_callback.activity_gated_start.mock_calls[0], # pyright: ignore + { + "plan_name": "standalone_read_hardware_for_ispyb", + "subplan_name": "run_gridscan_move_and_tidy", + }, + ) + assert_event( + test_ispyb_callback.activity_gated_event.mock_calls[0], # pyright: ignore + { + "undulator_current_gap": undulator_test_value, + "synchrotron-synchrotron_mode": synchrotron_test_value.value, + "s4_slit_gaps_xgap": xgap_test_value, + "s4_slit_gaps_ygap": ygap_test_value, + "robot-barcode": "BARCODE", + }, + ) + assert_event( + test_ispyb_callback.activity_gated_event.mock_calls[1], # pyright: ignore + { + "attenuator_actual_transmission": transmission_test_value, + "flux_flux_reading": flux_test_value, + }, + ) + # fmt: on @patch( "dodal.devices.aperturescatterguard.ApertureScatterguard._safe_move_within_datacollection_range", diff --git a/tests/unit_tests/external_interaction/callbacks/rotation/test_ispyb_callback.py b/tests/unit_tests/external_interaction/callbacks/rotation/test_ispyb_callback.py index b26e1453d..8b18765d8 100644 --- a/tests/unit_tests/external_interaction/callbacks/rotation/test_ispyb_callback.py +++ b/tests/unit_tests/external_interaction/callbacks/rotation/test_ispyb_callback.py @@ -7,7 +7,6 @@ from ...conftest import ( EXPECTED_END_TIME, EXPECTED_START_TIME, - TEST_BARCODE, TEST_DATA_COLLECTION_GROUP_ID, TEST_DATA_COLLECTION_IDS, TEST_SAMPLE_ID, @@ -27,11 +26,8 @@ "axisend": 180, "focal_spot_size_at_samplex": 1.0, "focal_spot_size_at_sampley": 1.0, - "slitgap_vertical": 1, - "slitgap_horizontal": 1, "beamsize_at_samplex": 1, "beamsize_at_sampley": 1, - "transmission": 100.0, "comments": "Hyperion rotation scan", "data_collection_number": 0, "detector_distance": 100.0, @@ -41,11 +37,9 @@ "imgsuffix": "h5", "n_passes": 1, "overlap": 0, - "flux": 10.0, "omegastart": 0, "start_image_number": 1, "resolution": 1.0, # deferred - "wavelength": 123.98419840550369, "xbeam": 150.0, "ybeam": 160.0, "xtal_snapshot1": "test_1_y", @@ -75,7 +69,6 @@ def test_activity_gated_start(mock_ispyb_conn, test_rotation_start_outer_documen "parentid": TEST_SESSION_ID, "experimenttype": "SAD", "sampleid": TEST_SAMPLE_ID, - "sample_barcode": TEST_BARCODE, # deferred }, ) assert_upsert_call_with( diff --git a/tests/unit_tests/external_interaction/callbacks/rotation/test_ispyb_mapping.py b/tests/unit_tests/external_interaction/callbacks/rotation/test_ispyb_mapping.py new file mode 100644 index 000000000..7e5c8216f --- /dev/null +++ b/tests/unit_tests/external_interaction/callbacks/rotation/test_ispyb_mapping.py @@ -0,0 +1,20 @@ +from unittest.mock import patch + +from hyperion.external_interaction.callbacks.rotation.ispyb_mapping import ( + populate_data_collection_info_for_rotation, +) + + +def test_populate_data_collection_info_for_rotation_checks_snapshots( + dummy_rotation_params, +): + with patch("hyperion.log.ISPYB_LOGGER.warning", autospec=True) as warning: + dummy_rotation_params.hyperion_params.ispyb_params.xtal_snapshots_omega_start = ( + None + ) + populate_data_collection_info_for_rotation( + dummy_rotation_params.hyperion_params.ispyb_params, + dummy_rotation_params.hyperion_params.detector_params, + dummy_rotation_params, + ) + warning.assert_called_once_with("No xtal snapshot paths sent to ISPyB!") diff --git a/tests/unit_tests/external_interaction/callbacks/test_rotation_callbacks.py b/tests/unit_tests/external_interaction/callbacks/test_rotation_callbacks.py index b4bbcf04d..05e248fc6 100644 --- a/tests/unit_tests/external_interaction/callbacks/test_rotation_callbacks.py +++ b/tests/unit_tests/external_interaction/callbacks/test_rotation_callbacks.py @@ -139,6 +139,9 @@ def activated_mocked_cbs(): "hyperion.external_interaction.callbacks.rotation.ispyb_callback.StoreInIspyb", autospec=True, ) +@pytest.mark.skip( + reason="TODO 1173 fix this test and nexus_callback implementation so that it doesn't assume that hw info is available at activity_gated_start()" +) def test_nexus_handler_gets_documents_in_mock_plan( ispyb, RE: RunEngine, diff --git a/tests/unit_tests/external_interaction/callbacks/xray_centre/test_ispyb_callback.py b/tests/unit_tests/external_interaction/callbacks/xray_centre/test_ispyb_callback.py index 6b156f230..73af6913f 100644 --- a/tests/unit_tests/external_interaction/callbacks/xray_centre/test_ispyb_callback.py +++ b/tests/unit_tests/external_interaction/callbacks/xray_centre/test_ispyb_callback.py @@ -6,8 +6,8 @@ from ...conftest import ( EXPECTED_START_TIME, - TEST_BARCODE, TEST_DATA_COLLECTION_GROUP_ID, + TEST_DATA_COLLECTION_IDS, TEST_SAMPLE_ID, TEST_SESSION_ID, assert_upsert_call_with, @@ -15,6 +15,84 @@ ) from ..conftest import TestData +EXPECTED_DATA_COLLECTION_3D = { + "visitid": TEST_SESSION_ID, + "parentid": TEST_DATA_COLLECTION_GROUP_ID, + "sampleid": TEST_SAMPLE_ID, + "detectorid": 78, + "axisstart": 0.0, + "axisrange": 0, + "axisend": 0, + "focal_spot_size_at_samplex": 0.0, + "focal_spot_size_at_sampley": 0.0, + "beamsize_at_samplex": 0.1, + "beamsize_at_sampley": 0.1, + "comments": "Hyperion: Xray centring - Diffraction grid scan of 40 by 20 " + "images in 100.0 um by 100.0 um steps. Top left (px): [50,100], " + "bottom right (px): [3250,1700].", + "data_collection_number": 0, + "detector_distance": 100.0, + "exp_time": 0.1, + "imgdir": "/tmp/", + "imgprefix": "file_name", + "imgsuffix": "h5", + "n_passes": 1, + "overlap": 0, + "omegastart": 0, + "start_image_number": 1, + "resolution": 1.0, # deferred + "wavelength": None, + "xbeam": 150.0, + "ybeam": 160.0, + "xtal_snapshot1": "test_1_y", + "xtal_snapshot2": "test_2_y", + "xtal_snapshot3": "test_3_y", + "synchrotron_mode": None, + "undulator_gap1": None, + "starttime": EXPECTED_START_TIME, + "filetemplate": "file_name_0_master.h5", + "nimages": 40 * 20, +} + +EXPECTED_DATA_COLLECTION_2D = { + "visitid": TEST_SESSION_ID, + "parentid": TEST_DATA_COLLECTION_GROUP_ID, + "sampleid": TEST_SAMPLE_ID, + "detectorid": 78, + "axisstart": 0.0, + "axisrange": 0, + "axisend": 0, + "focal_spot_size_at_samplex": 0.0, + "focal_spot_size_at_sampley": 0.0, + "beamsize_at_samplex": 0.1, + "beamsize_at_sampley": 0.1, + "comments": "Hyperion: Xray centring - Diffraction grid scan of 40 by 20 " + "images in 100.0 um by 100.0 um steps. Top left (px): [50,100], " + "bottom right (px): [450,300].", + "data_collection_number": 0, + "detector_distance": 100.0, + "exp_time": 0.1, + "imgdir": "/tmp/", + "imgprefix": "file_name", + "imgsuffix": "h5", + "n_passes": 1, + "overlap": 0, + "omegastart": 0, + "start_image_number": 1, + "resolution": 1.0, # deferred + "wavelength": None, + "xbeam": 150.0, + "ybeam": 160.0, + "xtal_snapshot1": "test_1_y", + "xtal_snapshot2": "test_2_y", + "xtal_snapshot3": "test_3_y", + "synchrotron_mode": None, + "undulator_gap1": None, + "starttime": EXPECTED_START_TIME, + "filetemplate": "file_name_0_master.h5", + "nimages": 40 * 20, +} + @patch( "hyperion.external_interaction.callbacks.common.ispyb_mapping.get_current_time_string", @@ -33,57 +111,91 @@ def test_activity_gated_start_2d(mock_ispyb_conn): "parentid": TEST_SESSION_ID, "experimenttype": "mesh", "sampleid": TEST_SAMPLE_ID, - "sample_barcode": TEST_BARCODE, # deferred }, ) assert_upsert_call_with( mx_acq.upsert_data_collection.mock_calls[0], mx_acq.get_data_collection_params(), + EXPECTED_DATA_COLLECTION_2D, + ) + mx_acq.upsert_data_collection.update_dc_position.assert_not_called() + mx_acq.upsert_data_collection.upsert_dc_grid.assert_not_called() + + +@patch( + "hyperion.external_interaction.callbacks.common.ispyb_mapping.get_current_time_string", + new=MagicMock(return_value=EXPECTED_START_TIME), +) +def test_activity_gated_event_2d(mock_ispyb_conn): + callback = GridscanISPyBCallback() + callback.activity_gated_start( + TestData.test_gridscan2d_start_document # pyright: ignore + ) + mx_acq = mx_acquisition_from_conn(mock_ispyb_conn) + mx_acq.upsert_data_collection_group.reset_mock() + mx_acq.upsert_data_collection.reset_mock() + callback.activity_gated_descriptor( + TestData.test_descriptor_document_pre_data_collection + ) + callback.activity_gated_event(TestData.test_event_document_pre_data_collection) + callback.activity_gated_descriptor( + TestData.test_descriptor_document_during_data_collection + ) + callback.activity_gated_event(TestData.test_event_document_during_data_collection) + + assert_upsert_call_with( + mx_acq.upsert_data_collection_group.mock_calls[0], # pyright: ignore + mx_acq.get_data_collection_group_params(), { - "visitid": TEST_SESSION_ID, - "parentid": TEST_DATA_COLLECTION_GROUP_ID, + "id": TEST_DATA_COLLECTION_GROUP_ID, + "parentid": TEST_SESSION_ID, + "experimenttype": "mesh", "sampleid": TEST_SAMPLE_ID, - "detectorid": 78, - "axisstart": 0.0, - "axisrange": 0, - "axisend": 0, - "focal_spot_size_at_samplex": 0.0, - "focal_spot_size_at_sampley": 0.0, - "slitgap_vertical": 0.1, - "slitgap_horizontal": 0.1, - "beamsize_at_samplex": 0.1, - "beamsize_at_sampley": 0.1, - "transmission": 100.0, - "comments": "Hyperion: Xray centring - Diffraction grid scan of 40 by 20 " - "images in 100.0 um by 100.0 um steps. Top left (px): [50,100], " - "bottom right (px): [450,300].", - "data_collection_number": 0, - "detector_distance": 100.0, - "exp_time": 0.1, - "imgdir": "/tmp/", - "imgprefix": "file_name", - "imgsuffix": "h5", - "n_passes": 1, - "overlap": 0, - "flux": 10.0, - "omegastart": 0, - "start_image_number": 1, - "resolution": 1.0, # deferred - "wavelength": None, - "xbeam": 150.0, - "ybeam": 160.0, - "xtal_snapshot1": "test_1_y", - "xtal_snapshot2": "test_2_y", - "xtal_snapshot3": "test_3_y", - "synchrotron_mode": None, - "undulator_gap1": 1.0, - "starttime": EXPECTED_START_TIME, - "filetemplate": "file_name_0_master.h5", - "nimages": 40 * 20, + "sample_barcode": "BARCODE", # deferred + }, + ) + assert_upsert_call_with( + mx_acq.upsert_data_collection.mock_calls[0], + mx_acq.get_data_collection_params(), + EXPECTED_DATA_COLLECTION_2D + | { + "id": TEST_DATA_COLLECTION_IDS[0], + "slitgaphorizontal": 0.1234, + "slitgapvertical": 0.2345, + "synchrotronmode": "test", + "undulatorgap1": 1.234, + "wavelength": 1.1164718451643736, + "transmission": 100, + "flux": 10, + }, + ) + assert_upsert_call_with( + mx_acq.update_dc_position.mock_calls[0], + mx_acq.get_dc_position_params(), + { + "id": TEST_DATA_COLLECTION_IDS[0], + "pos_x": 0, + "pos_y": 0, + "pos_z": 0, + }, + ) + assert_upsert_call_with( + mx_acq.upsert_dc_grid.mock_calls[0], + mx_acq.get_dc_grid_params(), + { + "parentid": TEST_DATA_COLLECTION_IDS[0], + "dxinmm": 0.1, + "dyinmm": 0.1, + "stepsx": 40, + "stepsy": 20, + "micronsperpixelx": 10, + "micronsperpixely": 10, + "snapshotoffsetxpixel": 50, + "snapshotoffsetypixel": 100, + "orientation": "horizontal", + "snaked": True, }, ) - mx_acq.upsert_data_collection.update_dc_position.assert_not_called() - mx_acq.upsert_data_collection.update_dc_grid.assert_not_called() @patch( @@ -101,54 +213,113 @@ def test_activity_gated_start_3d(mock_ispyb_conn): "parentid": TEST_SESSION_ID, "experimenttype": "Mesh3D", "sampleid": TEST_SAMPLE_ID, - "sample_barcode": TEST_BARCODE, # deferred }, ) assert_upsert_call_with( mx_acq.upsert_data_collection.mock_calls[0], mx_acq.get_data_collection_params(), + EXPECTED_DATA_COLLECTION_3D, + ) + mx_acq.upsert_data_collection.update_dc_position.assert_not_called() + mx_acq.upsert_data_collection.upsert_dc_grid.assert_not_called() + + +@patch( + "hyperion.external_interaction.callbacks.common.ispyb_mapping.get_current_time_string", + new=MagicMock(return_value=EXPECTED_START_TIME), +) +def test_activity_gated_event_3d(mock_ispyb_conn): + callback = GridscanISPyBCallback() + callback.activity_gated_start(TestData.test_start_document) # pyright: ignore + mx_acq = mx_acquisition_from_conn(mock_ispyb_conn) + mx_acq.upsert_data_collection_group.reset_mock() + mx_acq.upsert_data_collection.reset_mock() + callback.activity_gated_descriptor( + TestData.test_descriptor_document_pre_data_collection + ) + callback.activity_gated_event(TestData.test_event_document_pre_data_collection) + callback.activity_gated_descriptor( + TestData.test_descriptor_document_during_data_collection + ) + callback.activity_gated_event(TestData.test_event_document_during_data_collection) + + assert_upsert_call_with( + mx_acq.upsert_data_collection_group.mock_calls[0], # pyright: ignore + mx_acq.get_data_collection_group_params(), { - "visitid": TEST_SESSION_ID, - "parentid": TEST_DATA_COLLECTION_GROUP_ID, + "id": TEST_DATA_COLLECTION_GROUP_ID, + "parentid": TEST_SESSION_ID, + "experimenttype": "Mesh3D", "sampleid": TEST_SAMPLE_ID, - "detectorid": 78, - "axisstart": 0.0, - "axisrange": 0, - "axisend": 0, - "focal_spot_size_at_samplex": 0.0, - "focal_spot_size_at_sampley": 0.0, - "slitgap_vertical": 0.1, - "slitgap_horizontal": 0.1, - "beamsize_at_samplex": 0.1, - "beamsize_at_sampley": 0.1, - "transmission": 100.0, - "comments": "Hyperion: Xray centring - Diffraction grid scan of 40 by 20 " - "images in 100.0 um by 100.0 um steps. Top left (px): [50,100], " - "bottom right (px): [3250,1700].", - "data_collection_number": 0, - "detector_distance": 100.0, - "exp_time": 0.1, - "imgdir": "/tmp/", - "imgprefix": "file_name", - "imgsuffix": "h5", - "n_passes": 1, - "overlap": 0, - "flux": 10.0, - "omegastart": 0, - "start_image_number": 1, - "resolution": 1.0, # deferred - "wavelength": None, - "xbeam": 150.0, - "ybeam": 160.0, - "xtal_snapshot1": "test_1_y", - "xtal_snapshot2": "test_2_y", - "xtal_snapshot3": "test_3_y", - "synchrotron_mode": None, - "undulator_gap1": 1.0, - "starttime": EXPECTED_START_TIME, - "filetemplate": "file_name_0_master.h5", - "nimages": 40 * 20, + "sample_barcode": "BARCODE", # deferred + }, + ) + assert_upsert_call_with( + mx_acq.upsert_data_collection.mock_calls[0], + mx_acq.get_data_collection_params(), + EXPECTED_DATA_COLLECTION_3D + | { + "id": TEST_DATA_COLLECTION_IDS[0], + "slitgaphorizontal": 0.1234, + "slitgapvertical": 0.2345, + "synchrotronmode": "test", + "undulatorgap1": 1.234, + "wavelength": 1.1164718451643736, + "transmission": 100, + "flux": 10, + }, + ) + assert_upsert_call_with( + mx_acq.update_dc_position.mock_calls[0], + mx_acq.get_dc_position_params(), + { + "id": TEST_DATA_COLLECTION_IDS[0], + "pos_x": 0, + "pos_y": 0, + "pos_z": 0, + }, + ) + assert_upsert_call_with( + mx_acq.upsert_dc_grid.mock_calls[0], + mx_acq.get_dc_grid_params(), + { + "parentid": TEST_DATA_COLLECTION_IDS[0], + "dxinmm": 0.1, + "dyinmm": 0.1, + "stepsx": 40, + "stepsy": 20, + "micronsperpixelx": 1.25, + "micronsperpixely": 1.25, + "snapshotoffsetxpixel": 50, + "snapshotoffsetypixel": 100, + "orientation": "horizontal", + "snaked": True, + }, + ) + assert_upsert_call_with( + mx_acq.update_dc_position.mock_calls[1], + mx_acq.get_dc_position_params(), + { + "id": TEST_DATA_COLLECTION_IDS[1], + "pos_x": 0, + "pos_y": 0, + "pos_z": 0, + }, + ) + assert_upsert_call_with( + mx_acq.upsert_dc_grid.mock_calls[1], + mx_acq.get_dc_grid_params(), + { + "parentid": TEST_DATA_COLLECTION_IDS[1], + "dxinmm": 0.1, + "dyinmm": 0.1, + "stepsx": 40, + "stepsy": 10, + "micronsperpixelx": 1.25, + "micronsperpixely": 1.25, + "snapshotoffsetxpixel": 50, + "snapshotoffsetypixel": 0, + "orientation": "horizontal", + "snaked": True, }, ) - mx_acq.upsert_data_collection.update_dc_position.assert_not_called() - mx_acq.upsert_data_collection.update_dc_grid.assert_not_called() diff --git a/tests/unit_tests/external_interaction/callbacks/xray_centre/test_ispyb_mapping.py b/tests/unit_tests/external_interaction/callbacks/xray_centre/test_ispyb_mapping.py index 152c2ac93..d26191940 100644 --- a/tests/unit_tests/external_interaction/callbacks/xray_centre/test_ispyb_mapping.py +++ b/tests/unit_tests/external_interaction/callbacks/xray_centre/test_ispyb_mapping.py @@ -1,8 +1,11 @@ +from unittest.mock import MagicMock, patch + import numpy as np import pytest from hyperion.external_interaction.callbacks.common.ispyb_mapping import GridScanInfo from hyperion.external_interaction.callbacks.xray_centre.ispyb_mapping import ( + construct_comment_for_gridscan, populate_xy_data_collection_info, ) from hyperion.parameters.plan_specific.gridscan_internal_params import ( @@ -10,7 +13,6 @@ ) from ...conftest import ( - TEST_BARCODE, TEST_SAMPLE_ID, default_raw_params, ) @@ -20,7 +22,6 @@ def dummy_params(): dummy_params = GridscanInternalParameters(**default_raw_params()) dummy_params.hyperion_params.ispyb_params.sample_id = TEST_SAMPLE_ID - dummy_params.hyperion_params.ispyb_params.sample_barcode = TEST_BARCODE dummy_params.hyperion_params.ispyb_params.upper_left = np.array([100, 100, 50]) dummy_params.hyperion_params.ispyb_params.microns_per_pixel_x = 1.25 dummy_params.hyperion_params.ispyb_params.microns_per_pixel_y = 1.25 @@ -47,3 +48,63 @@ def test_given_x_and_y_steps_different_from_total_images_when_grid_scan_stored_t ) assert actual.n_images == expected_number_of_steps + + +@patch("ispyb.open", autospec=True) +def test_ispyb_deposition_rounds_position_to_int( + mock_ispyb_conn: MagicMock, + dummy_params, +): + dummy_params.hyperion_params.ispyb_params.upper_left = np.array([0.01, 100, 50]) + + assert construct_comment_for_gridscan( + dummy_params, + dummy_params.hyperion_params.ispyb_params, + GridScanInfo(dummy_params.hyperion_params.ispyb_params.upper_left, 20, 0.1), + ) == ( + "Hyperion: Xray centring - Diffraction grid scan of 40 by 20 images " + "in 100.0 um by 100.0 um steps. Top left (px): [0,100], bottom right (px): [3200,1700]." + ) + + +@pytest.mark.parametrize( + ["raw", "rounded"], + [ + (0.0012345, "1.2"), + (0.020000000, "20.0"), + (0.01999999, "20.0"), + (0.015257, "15.3"), + (0.0001234, "0.1"), + (0.0017345, "1.7"), + (0.0019945, "2.0"), + ], +) +@patch( + "hyperion.external_interaction.callbacks.xray_centre.ispyb_mapping.oav_utils.bottom_right_from_top_left", + autospec=True, +) +def test_ispyb_deposition_rounds_box_size_int( + bottom_right_from_top_left: MagicMock, + dummy_params: GridscanInternalParameters, + raw, + rounded, +): + dummy_params.experiment_params.x_steps = 0 + dummy_params.experiment_params.x_step_size = raw + grid_scan_info = GridScanInfo( + [ + 0, + 0, + 0, + ], + 0, + raw, + ) + bottom_right_from_top_left.return_value = grid_scan_info.upper_left + + assert construct_comment_for_gridscan( + dummy_params, MagicMock(), grid_scan_info + ) == ( + "Hyperion: Xray centring - Diffraction grid scan of 0 by 0 images in " + f"{rounded} um by {rounded} um steps. Top left (px): [0,0], bottom right (px): [0,0]." + ) diff --git a/tests/unit_tests/external_interaction/conftest.py b/tests/unit_tests/external_interaction/conftest.py index c02303dc4..4430c4cc0 100644 --- a/tests/unit_tests/external_interaction/conftest.py +++ b/tests/unit_tests/external_interaction/conftest.py @@ -69,7 +69,7 @@ def test_rotation_params(): ) param_dict["hyperion_params"]["detector_params"]["directory"] = "tests/test_data" param_dict["hyperion_params"]["detector_params"]["prefix"] = "TEST_FILENAME" - param_dict["hyperion_params"]["detector_params"]["current_energy_ev"] = 12700 + param_dict["hyperion_params"]["detector_params"]["expected_energy_ev"] = 12700 param_dict["hyperion_params"]["ispyb_params"]["current_energy_ev"] = 12700 param_dict["experiment_params"]["rotation_angle"] = 360.0 params = RotationInternalParameters(**param_dict) @@ -196,7 +196,6 @@ def dummy_rotation_params(): ) ) dummy_params.hyperion_params.ispyb_params.sample_id = TEST_SAMPLE_ID - dummy_params.hyperion_params.ispyb_params.sample_barcode = TEST_BARCODE return dummy_params diff --git a/tests/unit_tests/external_interaction/ispyb/conftest.py b/tests/unit_tests/external_interaction/ispyb/conftest.py index 939bdd547..531a38195 100644 --- a/tests/unit_tests/external_interaction/ispyb/conftest.py +++ b/tests/unit_tests/external_interaction/ispyb/conftest.py @@ -3,14 +3,12 @@ import numpy as np import pytest -from hyperion.external_interaction.callbacks.common.ispyb_mapping import ( - GridScanInfo, - populate_data_collection_position_info, +from hyperion.external_interaction.ispyb.data_model import ( + DataCollectionGridInfo, + DataCollectionPositionInfo, + ExperimentType, ) -from hyperion.external_interaction.callbacks.xray_centre.ispyb_mapping import ( - populate_data_collection_grid_info, -) -from hyperion.external_interaction.ispyb.data_model import ExperimentType +from hyperion.external_interaction.ispyb.ispyb_dataclass import Orientation from hyperion.external_interaction.ispyb.ispyb_store import StoreInIspyb from hyperion.parameters.constants import CONST from hyperion.parameters.plan_specific.gridscan_internal_params import ( @@ -18,7 +16,6 @@ ) from ..conftest import ( - TEST_BARCODE, TEST_DATA_COLLECTION_GROUP_ID, TEST_SAMPLE_ID, default_raw_params, @@ -29,7 +26,6 @@ def dummy_params(): dummy_params = GridscanInternalParameters(**default_raw_params()) dummy_params.hyperion_params.ispyb_params.sample_id = TEST_SAMPLE_ID - dummy_params.hyperion_params.ispyb_params.sample_barcode = TEST_BARCODE dummy_params.hyperion_params.ispyb_params.upper_left = np.array([100, 100, 50]) dummy_params.hyperion_params.ispyb_params.microns_per_pixel_x = 1.25 dummy_params.hyperion_params.ispyb_params.microns_per_pixel_y = 1.25 @@ -38,7 +34,7 @@ def dummy_params(): @pytest.fixture -def dummy_3d_gridscan_ispyb(dummy_params): +def dummy_3d_gridscan_ispyb(): store_in_ispyb_3d = StoreInIspyb(CONST.SIM.ISPYB_CONFIG, ExperimentType.GRIDSCAN_3D) return store_in_ispyb_3d @@ -50,29 +46,31 @@ def dummy_rotation_ispyb(dummy_rotation_params): @pytest.fixture -def dummy_2d_gridscan_ispyb(dummy_params): +def dummy_2d_gridscan_ispyb(): return StoreInIspyb(CONST.SIM.ISPYB_CONFIG, ExperimentType.GRIDSCAN_2D) @pytest.fixture -def scan_xy_data_info_for_update(dummy_params, scan_data_info_for_begin): +def scan_xy_data_info_for_update(scan_data_info_for_begin): scan_data_info_for_update = deepcopy(scan_data_info_for_begin) - grid_scan_info = GridScanInfo( - dummy_params.hyperion_params.ispyb_params.upper_left, - dummy_params.experiment_params.y_steps, - dummy_params.experiment_params.y_step_size, - ) scan_data_info_for_update.data_collection_info.parent_id = ( TEST_DATA_COLLECTION_GROUP_ID ) - scan_data_info_for_update.data_collection_grid_info = ( - populate_data_collection_grid_info( - dummy_params, grid_scan_info, dummy_params.hyperion_params.ispyb_params - ) + scan_data_info_for_update.data_collection_info.synchrotron_mode = "test" + scan_data_info_for_update.data_collection_info.flux = 10 + scan_data_info_for_update.data_collection_grid_info = DataCollectionGridInfo( + dx_in_mm=0.1, + dy_in_mm=0.1, + steps_x=40, + steps_y=20, + microns_per_pixel_x=1.25, + microns_per_pixel_y=1.25, + snapshot_offset_x_pixel=100, + snapshot_offset_y_pixel=100, + orientation=Orientation.HORIZONTAL, + snaked=True, ) scan_data_info_for_update.data_collection_position_info = ( - populate_data_collection_position_info( - dummy_params.hyperion_params.ispyb_params - ) + DataCollectionPositionInfo(0, 0, 0) ) return scan_data_info_for_update diff --git a/tests/unit_tests/external_interaction/ispyb/test_gridscan_ispyb_store_2d.py b/tests/unit_tests/external_interaction/ispyb/test_gridscan_ispyb_store_2d.py deleted file mode 100644 index 64e60b82e..000000000 --- a/tests/unit_tests/external_interaction/ispyb/test_gridscan_ispyb_store_2d.py +++ /dev/null @@ -1,686 +0,0 @@ -from unittest.mock import MagicMock, patch - -import numpy as np -import pytest -from ispyb.sp.mxacquisition import MXAcquisition - -from hyperion.external_interaction.callbacks.common.ispyb_mapping import ( - GridScanInfo, - populate_data_collection_group, - populate_remaining_data_collection_info, -) -from hyperion.external_interaction.callbacks.xray_centre.ispyb_mapping import ( - construct_comment_for_gridscan, - populate_xy_data_collection_info, -) -from hyperion.external_interaction.ispyb.data_model import ScanDataInfo -from hyperion.external_interaction.ispyb.ispyb_store import ( - IspybIds, - StoreInIspyb, -) -from hyperion.parameters.plan_specific.gridscan_internal_params import ( - GridscanInternalParameters, -) - -from ..conftest import ( - TEST_BARCODE, - TEST_DATA_COLLECTION_GROUP_ID, - TEST_DATA_COLLECTION_IDS, - TEST_GRID_INFO_IDS, - TEST_POSITION_ID, - TEST_SAMPLE_ID, - TEST_SESSION_ID, - assert_upsert_call_with, - mx_acquisition_from_conn, -) - -EXPECTED_START_TIME = "2024-02-08 14:03:59" -EXPECTED_END_TIME = "2024-02-08 14:04:01" - -EMPTY_DATA_COLLECTION_PARAMS = { - "id": None, - "parentid": None, - "visitid": None, - "sampleid": None, - "detectorid": None, - "positionid": None, - "apertureid": None, - "datacollectionnumber": None, - "starttime": None, - "endtime": None, - "runstatus": None, - "axisstart": None, - "axisend": None, - "axisrange": None, - "overlap": None, - "nimages": None, - "startimagenumber": None, - "npasses": None, - "exptime": None, - "imgdir": None, - "imgprefix": None, - "imgsuffix": None, - "imgcontainersubpath": None, - "filetemplate": None, - "wavelength": None, - "resolution": None, - "detectordistance": None, - "xbeam": None, - "ybeam": None, - "comments": None, - "slitgapvertical": None, - "slitgaphorizontal": None, - "transmission_fraction": None, - "synchrotronmode": None, - "xtalsnapshot1": None, - "xtalsnapshot2": None, - "xtalsnapshot3": None, - "xtalsnapshot4": None, - "rotationaxis": None, - "phistart": None, - "kappastart": None, - "omegastart": None, - "resolutionatcorner": None, - "detector2theta": None, - "undulatorgap1": None, - "undulatorgap2": None, - "undulatorgap3": None, - "beamsizeatsamplex": None, - "beamsizeatsampley": None, - "avgtemperature": None, - "actualcenteringposition": None, - "beamshape": None, - "focalspotsizeatsamplex": None, - "focalspotsizeatsampley": None, - "polarisation": None, - "flux": None, - "processeddatafile": None, - "datfile": None, - "magnification": None, - "totalabsorbeddose": None, - "binning": None, - "particlediameter": None, - "boxsizectf": None, - "minresolution": None, - "mindefocus": None, - "maxdefocus": None, - "defocusstepsize": None, - "amountastigmatism": None, - "extractsize": None, - "bgradius": None, - "voltage": None, - "objaperture": None, - "c1aperture": None, - "c2aperture": None, - "c3aperture": None, - "c1lens": None, - "c2lens": None, - "c3lens": None, -} - - -@pytest.fixture -def ispyb_conn(base_ispyb_conn): - return base_ispyb_conn - - -@pytest.fixture -def dummy_collection_group_info(dummy_params): - return populate_data_collection_group( - "mesh", - dummy_params.hyperion_params.detector_params, - dummy_params.hyperion_params.ispyb_params, - ) - - -@pytest.fixture -@patch( - "hyperion.external_interaction.callbacks.common.ispyb_mapping.get_current_time_string", - new=MagicMock(return_value=EXPECTED_START_TIME), -) -def scan_data_info_for_begin(dummy_params): - grid_scan_info = GridScanInfo( - dummy_params.hyperion_params.ispyb_params.upper_left, - dummy_params.experiment_params.y_steps, - dummy_params.experiment_params.y_step_size, - ) - info = ScanDataInfo( - data_collection_info=populate_remaining_data_collection_info( - lambda: construct_comment_for_gridscan( - dummy_params, dummy_params.hyperion_params.ispyb_params, grid_scan_info - ), - None, - populate_xy_data_collection_info( - grid_scan_info, - dummy_params, - dummy_params.hyperion_params.ispyb_params, - dummy_params.hyperion_params.detector_params, - ), - dummy_params.hyperion_params.detector_params, - dummy_params.hyperion_params.ispyb_params, - ), - ) - return info - - -def test_begin_deposition( - mock_ispyb_conn, - dummy_2d_gridscan_ispyb, - dummy_params, - dummy_collection_group_info, - scan_data_info_for_begin, -): - assert dummy_2d_gridscan_ispyb.begin_deposition( - dummy_collection_group_info, scan_data_info_for_begin - ) == IspybIds( - data_collection_group_id=TEST_DATA_COLLECTION_GROUP_ID, - data_collection_ids=(TEST_DATA_COLLECTION_IDS[0],), - ) - - mx_acq: MagicMock = mx_acquisition_from_conn(mock_ispyb_conn) - assert_upsert_call_with( - mx_acq.upsert_data_collection_group.mock_calls[0], # pyright: ignore - mx_acq.get_data_collection_group_params(), - { - "parentid": TEST_SESSION_ID, - "experimenttype": "mesh", - "sampleid": TEST_SAMPLE_ID, - "sample_barcode": TEST_BARCODE, # deferred - }, - ) - assert_upsert_call_with( - mx_acq.upsert_data_collection.mock_calls[0], - mx_acq.get_data_collection_params(), - { - "visitid": TEST_SESSION_ID, - "parentid": TEST_DATA_COLLECTION_GROUP_ID, - "sampleid": TEST_SAMPLE_ID, - "detectorid": 78, - "axisstart": 0.0, - "axisrange": 0, - "axisend": 0, - "focal_spot_size_at_samplex": 0.0, - "focal_spot_size_at_sampley": 0.0, - "slitgap_vertical": 0.1, - "slitgap_horizontal": 0.1, - "beamsize_at_samplex": 0.1, - "beamsize_at_sampley": 0.1, - "transmission": 100.0, - "comments": "Hyperion: Xray centring - Diffraction grid scan of 40 by 20 " - "images in 100.0 um by 100.0 um steps. Top left (px): [100,100], " - "bottom right (px): [3300,1700].", - "data_collection_number": 0, - "detector_distance": 100.0, - "exp_time": 0.1, - "imgdir": "/tmp/", - "imgprefix": "file_name", - "imgsuffix": "h5", - "n_passes": 1, - "overlap": 0, - "flux": 10.0, - "omegastart": 0, - "start_image_number": 1, - "resolution": 1.0, # deferred - "wavelength": 123.98419840550369, - "xbeam": 150.0, - "ybeam": 160.0, - "xtal_snapshot1": "test_1_y", - "xtal_snapshot2": "test_2_y", - "xtal_snapshot3": "test_3_y", - "synchrotron_mode": None, - "undulator_gap1": 1.0, - "starttime": EXPECTED_START_TIME, - "filetemplate": "file_name_0_master.h5", - "nimages": 40 * 20, - }, - ) - mx_acq.upsert_data_collection.update_dc_position.assert_not_called() - mx_acq.upsert_data_collection.update_dc_grid.assert_not_called() - - -@patch( - "hyperion.external_interaction.callbacks.common.ispyb_mapping.get_current_time_string", - new=MagicMock(return_value=EXPECTED_START_TIME), -) -def test_update_deposition( - mock_ispyb_conn, - dummy_2d_gridscan_ispyb, - dummy_params, - dummy_collection_group_info, - scan_data_info_for_begin, - scan_xy_data_info_for_update, -): - ispyb_ids = dummy_2d_gridscan_ispyb.begin_deposition( - dummy_collection_group_info, scan_data_info_for_begin - ) - mx_acq = mx_acquisition_from_conn(mock_ispyb_conn) - mx_acq.upsert_data_collection_group.assert_called_once() - mx_acq.upsert_data_collection.assert_called_once() - assert dummy_2d_gridscan_ispyb.update_deposition( - ispyb_ids, dummy_collection_group_info, [scan_xy_data_info_for_update] - ) == IspybIds( - data_collection_group_id=TEST_DATA_COLLECTION_GROUP_ID, - data_collection_ids=(TEST_DATA_COLLECTION_IDS[0],), - grid_ids=(TEST_GRID_INFO_IDS[0],), - ) - - assert_upsert_call_with( - mx_acq.upsert_data_collection_group.mock_calls[1], - mx_acq.get_data_collection_group_params(), - { - "id": TEST_DATA_COLLECTION_GROUP_ID, - "parentid": TEST_SESSION_ID, - "experimenttype": "mesh", - "sampleid": TEST_SAMPLE_ID, - "sample_barcode": TEST_BARCODE, - }, - ) - - assert_upsert_call_with( - mx_acq.upsert_data_collection.mock_calls[1], - mx_acq.get_data_collection_params(), - { - "id": 12, - "visitid": TEST_SESSION_ID, - "parentid": TEST_DATA_COLLECTION_GROUP_ID, - "sampleid": TEST_SAMPLE_ID, - "detectorid": 78, - "axisstart": 0.0, - "axisrange": 0, - "axisend": 0, - "focal_spot_size_at_samplex": 0.0, - "focal_spot_size_at_sampley": 0.0, - "slitgap_vertical": 0.1, - "slitgap_horizontal": 0.1, - "beamsize_at_samplex": 0.1, - "beamsize_at_sampley": 0.1, - "transmission": 100.0, - "comments": "Hyperion: Xray centring - Diffraction grid scan of 40 by 20 " - "images in 100.0 um by 100.0 um steps. Top left (px): [100,100], " - "bottom right (px): [3300,1700].", - "data_collection_number": 0, - "detector_distance": 100.0, - "exp_time": 0.1, - "imgdir": "/tmp/", - "imgprefix": "file_name", - "imgsuffix": "h5", - "n_passes": 1, - "overlap": 0, - "flux": 10.0, - "omegastart": 0.0, - "start_image_number": 1, - "resolution": 1.0, # deferred - "wavelength": 123.98419840550369, - "xbeam": 150.0, - "ybeam": 160.0, - "xtal_snapshot1": "test_1_y", - "xtal_snapshot2": "test_2_y", - "xtal_snapshot3": "test_3_y", - "synchrotron_mode": None, - "undulator_gap1": 1.0, - "starttime": EXPECTED_START_TIME, - "filetemplate": "file_name_0_master.h5", - "nimages": 40 * 20, - }, - ) - - assert_upsert_call_with( - mx_acq.update_dc_position.mock_calls[0], - mx_acq.get_dc_position_params(), - { - "id": TEST_DATA_COLLECTION_IDS[0], - "pos_x": dummy_params.hyperion_params.ispyb_params.position[0], - "pos_y": dummy_params.hyperion_params.ispyb_params.position[1], - "pos_z": dummy_params.hyperion_params.ispyb_params.position[2], - }, - ) - - assert_upsert_call_with( - mx_acq.upsert_dc_grid.mock_calls[0], - mx_acq.get_dc_grid_params(), - { - "parentid": TEST_DATA_COLLECTION_IDS[0], - "dxinmm": dummy_params.experiment_params.x_step_size, - "dyinmm": dummy_params.experiment_params.y_step_size, - "stepsx": dummy_params.experiment_params.x_steps, - "stepsy": dummy_params.experiment_params.y_steps, - "micronsperpixelx": dummy_params.hyperion_params.ispyb_params.microns_per_pixel_x, - "micronsperpixely": dummy_params.hyperion_params.ispyb_params.microns_per_pixel_y, - "snapshotoffsetxpixel": dummy_params.hyperion_params.ispyb_params.upper_left[ - 0 - ], - "snapshotoffsetypixel": dummy_params.hyperion_params.ispyb_params.upper_left[ - 1 - ], - "orientation": "horizontal", - "snaked": True, - }, - ) - assert len(mx_acq.update_dc_position.mock_calls) == 1 - assert len(mx_acq.upsert_dc_grid.mock_calls) == 1 - assert len(mx_acq.upsert_data_collection.mock_calls) == 2 - assert len(mx_acq.upsert_data_collection_group.mock_calls) == 2 - - -@patch( - "hyperion.external_interaction.callbacks.common.ispyb_mapping.get_current_time_string", - new=MagicMock(return_value=EXPECTED_START_TIME), -) -@patch("hyperion.external_interaction.ispyb.ispyb_store.get_current_time_string") -def test_end_deposition_happy_path( - get_current_time, - mock_ispyb_conn, - dummy_2d_gridscan_ispyb, - dummy_params, - dummy_collection_group_info, - scan_data_info_for_begin, - scan_xy_data_info_for_update, -): - ispyb_ids = dummy_2d_gridscan_ispyb.begin_deposition( - dummy_collection_group_info, scan_data_info_for_begin - ) - ispyb_ids = dummy_2d_gridscan_ispyb.update_deposition( - ispyb_ids, dummy_collection_group_info, [scan_xy_data_info_for_update] - ) - mx_acq: MagicMock = mx_acquisition_from_conn(mock_ispyb_conn) - assert len(mx_acq.upsert_data_collection_group.mock_calls) == 2 - assert len(mx_acq.upsert_data_collection.mock_calls) == 2 - assert len(mx_acq.upsert_dc_grid.mock_calls) == 1 - - get_current_time.return_value = EXPECTED_END_TIME - dummy_2d_gridscan_ispyb.end_deposition(ispyb_ids, "success", "Test succeeded") - assert mx_acq.update_data_collection_append_comments.call_args_list[0] == ( - ( - TEST_DATA_COLLECTION_IDS[0], - "DataCollection Successful reason: Test succeeded", - " ", - ), - ) - assert_upsert_call_with( - mx_acq.upsert_data_collection.mock_calls[2], - mx_acq.get_data_collection_params(), - { - "id": TEST_DATA_COLLECTION_IDS[0], - "parentid": TEST_DATA_COLLECTION_GROUP_ID, - "endtime": EXPECTED_END_TIME, - "runstatus": "DataCollection Successful", - }, - ) - assert len(mx_acq.upsert_data_collection.mock_calls) == 3 - - -def setup_mock_return_values(ispyb_conn): - mx_acquisition = ispyb_conn.return_value.__enter__.return_value.mx_acquisition - - mx_acquisition.get_data_collection_group_params = ( - MXAcquisition.get_data_collection_group_params - ) - mx_acquisition.get_data_collection_params = MXAcquisition.get_data_collection_params - mx_acquisition.get_dc_grid_params = MXAcquisition.get_dc_grid_params - mx_acquisition.get_dc_position_params = MXAcquisition.get_dc_position_params - - ispyb_conn.return_value.core.retrieve_visit_id.return_value = TEST_SESSION_ID - mx_acquisition.upsert_data_collection.side_effect = TEST_DATA_COLLECTION_IDS * 2 - mx_acquisition.update_dc_position.return_value = TEST_POSITION_ID - mx_acquisition.upsert_data_collection_group.return_value = ( - TEST_DATA_COLLECTION_GROUP_ID - ) - mx_acquisition.upsert_dc_grid.return_value = TEST_GRID_INFO_IDS[0] - - -def test_param_keys( - mock_ispyb_conn, - dummy_2d_gridscan_ispyb, - dummy_params, - dummy_collection_group_info, - scan_data_info_for_begin, - scan_xy_data_info_for_update, -): - ispyb_ids = dummy_2d_gridscan_ispyb.begin_deposition( - dummy_collection_group_info, scan_data_info_for_begin - ) - assert dummy_2d_gridscan_ispyb.update_deposition( - ispyb_ids, dummy_collection_group_info, [scan_xy_data_info_for_update] - ) == IspybIds( - data_collection_ids=(TEST_DATA_COLLECTION_IDS[0],), - data_collection_group_id=TEST_DATA_COLLECTION_GROUP_ID, - grid_ids=(TEST_GRID_INFO_IDS[0],), - ) - - -def _test_when_grid_scan_stored_then_data_present_in_upserts( - ispyb_conn, - dummy_ispyb, - test_function, - dummy_collection_group_info, - scan_data_info_for_begin, - scan_data_info_for_update, - test_group=False, -): - setup_mock_return_values(ispyb_conn) - ispyb_ids = dummy_ispyb.begin_deposition( - dummy_collection_group_info, scan_data_info_for_begin - ) - dummy_ispyb.update_deposition( - ispyb_ids, dummy_collection_group_info, [scan_data_info_for_update] - ) - - mx_acquisition = ispyb_conn.return_value.__enter__.return_value.mx_acquisition - - upsert_data_collection_arg_list = ( - mx_acquisition.upsert_data_collection.call_args_list[1][0] - ) - actual = upsert_data_collection_arg_list[0] - assert test_function(MXAcquisition.get_data_collection_params(), actual) - - if test_group: - upsert_data_collection_group_arg_list = ( - mx_acquisition.upsert_data_collection_group.call_args_list[1][0] - ) - actual = upsert_data_collection_group_arg_list[0] - assert test_function(MXAcquisition.get_data_collection_group_params(), actual) - - -@patch("ispyb.open", autospec=True) -def test_given_sampleid_of_none_when_grid_scan_stored_then_sample_id_not_set( - ispyb_conn, - dummy_2d_gridscan_ispyb, - dummy_params, - dummy_collection_group_info, - scan_data_info_for_begin, - scan_xy_data_info_for_update, -): - dummy_params.hyperion_params.ispyb_params.sample_id = None - dummy_collection_group_info.sample_id = None - scan_data_info_for_begin.data_collection_info.sample_id = None - scan_xy_data_info_for_update.data_collection_info.sample_id = None - - def test_sample_id(default_params, actual): - sampleid_idx = list(default_params).index("sampleid") - return actual[sampleid_idx] == default_params["sampleid"] - - _test_when_grid_scan_stored_then_data_present_in_upserts( - ispyb_conn, - dummy_2d_gridscan_ispyb, - test_sample_id, - dummy_collection_group_info, - scan_data_info_for_begin, - scan_xy_data_info_for_update, - True, - ) - - -@patch("ispyb.open", autospec=True) -def test_given_real_sampleid_when_grid_scan_stored_then_sample_id_set( - ispyb_conn, - dummy_2d_gridscan_ispyb: StoreInIspyb, - dummy_params: GridscanInternalParameters, - dummy_collection_group_info, - scan_data_info_for_begin, - scan_xy_data_info_for_update, -): - expected_sample_id = "0001" - dummy_params.hyperion_params.ispyb_params.sample_id = expected_sample_id - - def test_sample_id(default_params, actual): - sampleid_idx = list(default_params).index("sampleid") - return actual[sampleid_idx] == expected_sample_id - - _test_when_grid_scan_stored_then_data_present_in_upserts( - ispyb_conn, - dummy_2d_gridscan_ispyb, - test_sample_id, - dummy_collection_group_info, - scan_data_info_for_begin, - scan_xy_data_info_for_update, - True, - ) - - -def test_fail_result_run_results_in_bad_run_status( - mock_ispyb_conn: MagicMock, - dummy_2d_gridscan_ispyb: StoreInIspyb, - dummy_params, - dummy_collection_group_info, - scan_data_info_for_begin, - scan_xy_data_info_for_update, -): - mock_ispyb_conn = mock_ispyb_conn - mock_mx_aquisition = ( - mock_ispyb_conn.return_value.__enter__.return_value.mx_acquisition - ) - mock_upsert_data_collection = mock_mx_aquisition.upsert_data_collection - - ispyb_ids = dummy_2d_gridscan_ispyb.begin_deposition( - dummy_collection_group_info, scan_data_info_for_begin - ) - ispyb_ids = dummy_2d_gridscan_ispyb.update_deposition( - ispyb_ids, dummy_collection_group_info, [scan_xy_data_info_for_update] - ) - dummy_2d_gridscan_ispyb.end_deposition(ispyb_ids, "fail", "test specifies failure") - - mock_upsert_data_collection_calls = mock_upsert_data_collection.call_args_list - end_deposition_upsert_args = mock_upsert_data_collection_calls[2][0] - upserted_param_value_list = end_deposition_upsert_args[0] - assert "DataCollection Unsuccessful" in upserted_param_value_list - assert "DataCollection Successful" not in upserted_param_value_list - - -def test_no_exception_during_run_results_in_good_run_status( - mock_ispyb_conn: MagicMock, - dummy_2d_gridscan_ispyb: StoreInIspyb, - dummy_params, - dummy_collection_group_info, - scan_data_info_for_begin, - scan_xy_data_info_for_update, -): - mock_ispyb_conn = mock_ispyb_conn - setup_mock_return_values(mock_ispyb_conn) - mock_mx_aquisition = ( - mock_ispyb_conn.return_value.__enter__.return_value.mx_acquisition - ) - mock_upsert_data_collection = mock_mx_aquisition.upsert_data_collection - - ispyb_ids = dummy_2d_gridscan_ispyb.begin_deposition( - dummy_collection_group_info, scan_data_info_for_begin - ) - ispyb_ids = dummy_2d_gridscan_ispyb.update_deposition( - ispyb_ids, dummy_collection_group_info, [scan_xy_data_info_for_update] - ) - dummy_2d_gridscan_ispyb.end_deposition(ispyb_ids, "success", "") - - mock_upsert_data_collection_calls = mock_upsert_data_collection.call_args_list - end_deposition_upsert_args = mock_upsert_data_collection_calls[2][0] - upserted_param_value_list = end_deposition_upsert_args[0] - assert "DataCollection Unsuccessful" not in upserted_param_value_list - assert "DataCollection Successful" in upserted_param_value_list - - -def test_ispyb_deposition_comment_correct( - mock_ispyb_conn: MagicMock, - dummy_2d_gridscan_ispyb: StoreInIspyb, - dummy_params, - dummy_collection_group_info, - scan_data_info_for_begin, - scan_xy_data_info_for_update, -): - mock_mx_aquisition = mock_ispyb_conn.return_value.mx_acquisition - mock_upsert_data_collection = mock_mx_aquisition.upsert_data_collection - ispyb_ids = dummy_2d_gridscan_ispyb.begin_deposition( - dummy_collection_group_info, scan_data_info_for_begin - ) - dummy_2d_gridscan_ispyb.update_deposition( - ispyb_ids, dummy_collection_group_info, [scan_xy_data_info_for_update] - ) - mock_upsert_call_args = mock_upsert_data_collection.call_args_list[0][0] - - upserted_param_value_list = mock_upsert_call_args[0] - assert upserted_param_value_list[29] == ( - "Hyperion: Xray centring - Diffraction grid scan of 40 by 20 images " - "in 100.0 um by 100.0 um steps. Top left (px): [100,100], bottom right (px): [3300,1700]." - ) - - -@patch("ispyb.open", autospec=True) -def test_ispyb_deposition_rounds_position_to_int( - mock_ispyb_conn: MagicMock, - dummy_2d_gridscan_ispyb: StoreInIspyb, - dummy_params, - dummy_collection_group_info, - scan_data_info_for_begin, - scan_xy_data_info_for_update, -): - dummy_params.hyperion_params.ispyb_params.upper_left = np.array([0.01, 100, 50]) - - assert construct_comment_for_gridscan( - dummy_params, - dummy_params.hyperion_params.ispyb_params, - GridScanInfo(dummy_params.hyperion_params.ispyb_params.upper_left, 20, 0.1), - ) == ( - "Hyperion: Xray centring - Diffraction grid scan of 40 by 20 images " - "in 100.0 um by 100.0 um steps. Top left (px): [0,100], bottom right (px): [3200,1700]." - ) - - -@pytest.mark.parametrize( - ["raw", "rounded"], - [ - (0.0012345, "1.2"), - (0.020000000, "20.0"), - (0.01999999, "20.0"), - (0.015257, "15.3"), - (0.0001234, "0.1"), - (0.0017345, "1.7"), - (0.0019945, "2.0"), - ], -) -@patch( - "hyperion.external_interaction.callbacks.xray_centre.ispyb_mapping.oav_utils.bottom_right_from_top_left", - autospec=True, -) -def test_ispyb_deposition_rounds_box_size_int( - bottom_right_from_top_left: MagicMock, - dummy_2d_gridscan_ispyb: StoreInIspyb, - dummy_params: GridscanInternalParameters, - raw, - rounded, -): - dummy_params.experiment_params.x_steps = 0 - dummy_params.experiment_params.x_step_size = raw - grid_scan_info = GridScanInfo( - [ - 0, - 0, - 0, - ], - 0, - raw, - ) - bottom_right_from_top_left.return_value = grid_scan_info.upper_left - - assert construct_comment_for_gridscan( - dummy_params, MagicMock(), grid_scan_info - ) == ( - "Hyperion: Xray centring - Diffraction grid scan of 0 by 0 images in " - f"{rounded} um by {rounded} um steps. Top left (px): [0,0], bottom right (px): [0,0]." - ) diff --git a/tests/unit_tests/external_interaction/ispyb/test_gridscan_ispyb_store_3d.py b/tests/unit_tests/external_interaction/ispyb/test_gridscan_ispyb_store_3d.py index 6c3c41f30..328b0b18f 100644 --- a/tests/unit_tests/external_interaction/ispyb/test_gridscan_ispyb_store_3d.py +++ b/tests/unit_tests/external_interaction/ispyb/test_gridscan_ispyb_store_3d.py @@ -1,34 +1,27 @@ from unittest.mock import MagicMock, patch -import numpy as np import pytest - -from hyperion.external_interaction.callbacks.common.ispyb_mapping import ( - GridScanInfo, - populate_data_collection_group, - populate_data_collection_position_info, - populate_remaining_data_collection_info, -) -from hyperion.external_interaction.callbacks.xray_centre.ispyb_mapping import ( - construct_comment_for_gridscan, - populate_data_collection_grid_info, - populate_xy_data_collection_info, - populate_xz_data_collection_info, +from ispyb.sp.mxacquisition import MXAcquisition + +from hyperion.external_interaction.ispyb.data_model import ( + DataCollectionGridInfo, + DataCollectionGroupInfo, + DataCollectionInfo, + DataCollectionPositionInfo, + ScanDataInfo, ) -from hyperion.external_interaction.ispyb.data_model import ScanDataInfo +from hyperion.external_interaction.ispyb.ispyb_dataclass import Orientation from hyperion.external_interaction.ispyb.ispyb_store import ( IspybIds, StoreInIspyb, ) -from hyperion.parameters.plan_specific.gridscan_internal_params import ( - GridscanInternalParameters, -) from ..conftest import ( TEST_BARCODE, TEST_DATA_COLLECTION_GROUP_ID, TEST_DATA_COLLECTION_IDS, TEST_GRID_INFO_IDS, + TEST_POSITION_ID, TEST_SAMPLE_ID, TEST_SESSION_ID, assert_upsert_call_with, @@ -40,110 +33,208 @@ @pytest.fixture -def dummy_params_3d(dummy_params): - x = 50 - y = 100 - z = 120 - dummy_params.hyperion_params.ispyb_params.upper_left = np.array([x, y, z]) - dummy_params.experiment_params.z_step_size = 0.2 - return dummy_params - - -@pytest.fixture -def dummy_collection_group_info(dummy_params_3d): - return populate_data_collection_group( - "Mesh3D", - dummy_params_3d.hyperion_params.detector_params, - dummy_params_3d.hyperion_params.ispyb_params, +def dummy_collection_group_info(): + return DataCollectionGroupInfo( + visit_string="cm31105-4", + experiment_type="Mesh3D", + sample_id="0001", ) @pytest.fixture -@patch( - "hyperion.external_interaction.callbacks.common.ispyb_mapping.get_current_time_string", - new=MagicMock(return_value=EXPECTED_START_TIME), -) -def scan_data_info_for_begin(dummy_params_3d): - grid_scan_info = GridScanInfo( - dummy_params_3d.hyperion_params.ispyb_params.upper_left, - dummy_params_3d.experiment_params.y_steps, - dummy_params_3d.experiment_params.y_step_size, - ) +def scan_data_info_for_begin(): return ScanDataInfo( - data_collection_info=populate_remaining_data_collection_info( - lambda: construct_comment_for_gridscan( - dummy_params_3d, - dummy_params_3d.hyperion_params.ispyb_params, - grid_scan_info, - ), - None, - populate_xy_data_collection_info( - grid_scan_info, - dummy_params_3d, - dummy_params_3d.hyperion_params.ispyb_params, - dummy_params_3d.hyperion_params.detector_params, - ), - dummy_params_3d.hyperion_params.detector_params, - dummy_params_3d.hyperion_params.ispyb_params, + data_collection_info=DataCollectionInfo( + omega_start=0.0, + data_collection_number=0, + xtal_snapshot1="test_1_y", + xtal_snapshot2="test_2_y", + xtal_snapshot3="test_3_y", + n_images=800, + axis_range=0, + axis_end=0.0, + kappa_start=None, + parent_id=None, + visit_string="cm31105-4", + sample_id="0001", + detector_id=78, + axis_start=0.0, + focal_spot_size_at_samplex=0.0, + focal_spot_size_at_sampley=0.0, + slitgap_vertical=0.1, + slitgap_horizontal=0.1, + beamsize_at_samplex=0.1, + beamsize_at_sampley=0.1, + transmission=100.0, + comments="Hyperion: Xray centring - Diffraction grid scan of 40 by 20 images in 100.0 um by 100.0 um steps. Top left (px): [50,100], bottom right (px): [3250,1700].", + detector_distance=100.0, + exp_time=0.1, + imgdir="/tmp/", + file_template="file_name_0_master.h5", + imgprefix="file_name", + imgsuffix="h5", + n_passes=1, + overlap=0, + start_image_number=1, + resolution=1.0, + wavelength=123.98419840550369, + xbeam=150.0, + ybeam=160.0, + synchrotron_mode=None, + undulator_gap1=1.0, + start_time=EXPECTED_START_TIME, ), + data_collection_id=None, + data_collection_position_info=None, + data_collection_grid_info=None, ) @pytest.fixture -@patch( - "hyperion.external_interaction.callbacks.common.ispyb_mapping.get_current_time_string", - new=MagicMock(return_value=EXPECTED_START_TIME), -) -def scan_data_infos_for_update(scan_xy_data_info_for_update, dummy_params): - upper_left = dummy_params.hyperion_params.ispyb_params.upper_left - xz_grid_scan_info = GridScanInfo( - [upper_left[0], upper_left[2]], - dummy_params.experiment_params.z_steps, - dummy_params.experiment_params.z_step_size, - ) - xz_data_collection_info = populate_xz_data_collection_info( - xz_grid_scan_info, - dummy_params, - dummy_params.hyperion_params.ispyb_params, - dummy_params.hyperion_params.detector_params, - ) - - def comment_constructor(): - return construct_comment_for_gridscan( - dummy_params, dummy_params.hyperion_params.ispyb_params, xz_grid_scan_info - ) - - xz_data_collection_info = populate_remaining_data_collection_info( - comment_constructor, - TEST_DATA_COLLECTION_GROUP_ID, - xz_data_collection_info, - dummy_params.hyperion_params.detector_params, - dummy_params.hyperion_params.ispyb_params, +def scan_data_infos_for_update(): + scan_xy_data_info_for_update = ScanDataInfo( + data_collection_info=DataCollectionInfo( + omega_start=0.0, + data_collection_number=0, + xtal_snapshot1="test_1_y", + xtal_snapshot2="test_2_y", + xtal_snapshot3="test_3_y", + n_images=800, + axis_range=0, + axis_end=0.0, + kappa_start=None, + parent_id=34, + visit_string="cm31105-4", + sample_id="0001", + detector_id=78, + axis_start=0.0, + focal_spot_size_at_samplex=0.0, + focal_spot_size_at_sampley=0.0, + slitgap_vertical=0.1, + slitgap_horizontal=0.1, + beamsize_at_samplex=0.1, + beamsize_at_sampley=0.1, + transmission=100.0, + comments="Hyperion: Xray centring - Diffraction grid scan of 40 by 20 images in 100.0 um by 100.0 um steps. Top left (px): [50,100], bottom right (px): [3250,1700].", + detector_distance=100.0, + exp_time=0.1, + imgdir="/tmp/", + file_template="file_name_0_master.h5", + imgprefix="file_name", + imgsuffix="h5", + n_passes=1, + overlap=0, + flux=10.0, + start_image_number=1, + resolution=1.0, + wavelength=123.98419840550369, + xbeam=150.0, + ybeam=160.0, + synchrotron_mode="test", + undulator_gap1=1.0, + start_time=EXPECTED_START_TIME, + ), + data_collection_id=None, + data_collection_position_info=DataCollectionPositionInfo( + pos_x=0, pos_y=0, pos_z=0 + ), + data_collection_grid_info=DataCollectionGridInfo( + dx_in_mm=0.1, + dy_in_mm=0.1, + steps_x=40, + steps_y=20, + microns_per_pixel_x=1.25, + microns_per_pixel_y=1.25, + snapshot_offset_x_pixel=50, + snapshot_offset_y_pixel=100, + orientation=Orientation.HORIZONTAL, + snaked=True, + ), ) - xz_data_collection_info.parent_id = TEST_DATA_COLLECTION_GROUP_ID - scan_xz_data_info_for_update = ScanDataInfo( - data_collection_info=xz_data_collection_info, - data_collection_grid_info=( - populate_data_collection_grid_info( - dummy_params, - xz_grid_scan_info, - dummy_params.hyperion_params.ispyb_params, - ) + data_collection_info=DataCollectionInfo( + omega_start=90.0, + data_collection_number=1, + xtal_snapshot1="test_1_z", + xtal_snapshot2="test_2_z", + xtal_snapshot3="test_3_z", + n_images=400, + axis_range=0, + axis_end=90.0, + kappa_start=None, + parent_id=34, + visit_string="cm31105-4", + sample_id="0001", + detector_id=78, + axis_start=90.0, + focal_spot_size_at_samplex=0.0, + focal_spot_size_at_sampley=0.0, + slitgap_vertical=0.1, + slitgap_horizontal=0.1, + beamsize_at_samplex=0.1, + beamsize_at_sampley=0.1, + transmission=100.0, + comments="Hyperion: Xray centring - Diffraction grid scan of 40 by 10 images in 100.0 um by 200.0 um steps. Top left (px): [50,120], bottom right (px): [3250,1720].", + detector_distance=100.0, + exp_time=0.1, + imgdir="/tmp/", + file_template="file_name_1_master.h5", + imgprefix="file_name", + imgsuffix="h5", + n_passes=1, + overlap=0, + flux=10.0, + start_image_number=1, + resolution=1.0, + wavelength=123.98419840550369, + xbeam=150.0, + ybeam=160.0, + synchrotron_mode="test", + undulator_gap1=1.0, + start_time=EXPECTED_START_TIME, + ), + data_collection_id=None, + data_collection_position_info=DataCollectionPositionInfo( + pos_x=0.0, pos_y=0.0, pos_z=0.0 ), - data_collection_position_info=( - populate_data_collection_position_info( - dummy_params.hyperion_params.ispyb_params - ) + data_collection_grid_info=DataCollectionGridInfo( + dx_in_mm=0.1, + dy_in_mm=0.2, + steps_x=40, + steps_y=10, + microns_per_pixel_x=1.25, + microns_per_pixel_y=1.25, + snapshot_offset_x_pixel=50, + snapshot_offset_y_pixel=120, + orientation=Orientation.HORIZONTAL, + snaked=True, ), ) return [scan_xy_data_info_for_update, scan_xz_data_info_for_update] +def setup_mock_return_values(ispyb_conn): + mx_acquisition = ispyb_conn.return_value.__enter__.return_value.mx_acquisition + + mx_acquisition.get_data_collection_group_params = ( + MXAcquisition.get_data_collection_group_params + ) + mx_acquisition.get_data_collection_params = MXAcquisition.get_data_collection_params + mx_acquisition.get_dc_grid_params = MXAcquisition.get_dc_grid_params + mx_acquisition.get_dc_position_params = MXAcquisition.get_dc_position_params + + ispyb_conn.return_value.core.retrieve_visit_id.return_value = TEST_SESSION_ID + mx_acquisition.upsert_data_collection.side_effect = TEST_DATA_COLLECTION_IDS * 2 + mx_acquisition.update_dc_position.return_value = TEST_POSITION_ID + mx_acquisition.upsert_data_collection_group.return_value = ( + TEST_DATA_COLLECTION_GROUP_ID + ) + mx_acquisition.upsert_dc_grid.return_value = TEST_GRID_INFO_IDS[0] + + def test_ispyb_deposition_comment_for_3D_correct( mock_ispyb_conn: MagicMock, dummy_3d_gridscan_ispyb: StoreInIspyb, - dummy_params_3d, dummy_collection_group_info, scan_data_info_for_begin, scan_data_infos_for_update, @@ -173,7 +264,6 @@ def test_ispyb_deposition_comment_for_3D_correct( def test_store_3d_grid_scan( mock_ispyb_conn, dummy_3d_gridscan_ispyb: StoreInIspyb, - dummy_params_3d: GridscanInternalParameters, dummy_collection_group_info, scan_data_info_for_begin, scan_data_infos_for_update, @@ -204,7 +294,6 @@ def test_store_3d_grid_scan( def test_begin_deposition( mock_ispyb_conn, dummy_3d_gridscan_ispyb: StoreInIspyb, - dummy_params_3d: GridscanInternalParameters, dummy_collection_group_info, scan_data_info_for_begin, ): @@ -223,7 +312,6 @@ def test_begin_deposition( "parentid": TEST_SESSION_ID, "experimenttype": "Mesh3D", "sampleid": TEST_SAMPLE_ID, - "sample_barcode": TEST_BARCODE, # deferred }, ) mx_acq.upsert_data_collection.assert_called_once() @@ -256,7 +344,6 @@ def test_begin_deposition( "imgsuffix": "h5", "n_passes": 1, "overlap": 0, - "flux": 10.0, "omegastart": 0, "start_image_number": 1, "resolution": 1.0, # deferred @@ -284,7 +371,6 @@ def test_begin_deposition( def test_update_deposition( mock_ispyb_conn, dummy_3d_gridscan_ispyb, - dummy_params_3d, dummy_collection_group_info, scan_data_info_for_begin, scan_data_infos_for_update, @@ -296,6 +382,8 @@ def test_update_deposition( mx_acq.upsert_data_collection_group.assert_called_once() mx_acq.upsert_data_collection.assert_called_once() + dummy_collection_group_info.sample_barcode = TEST_BARCODE + actual_rows = dummy_3d_gridscan_ispyb.update_deposition( ispyb_ids, dummy_collection_group_info, scan_data_infos_for_update ) @@ -314,7 +402,7 @@ def test_update_deposition( "parentid": TEST_SESSION_ID, "experimenttype": "Mesh3D", "sampleid": TEST_SAMPLE_ID, - "sample_barcode": TEST_BARCODE, # deferred + "sample_barcode": TEST_BARCODE, }, ) @@ -358,7 +446,7 @@ def test_update_deposition( "xtal_snapshot1": "test_1_y", "xtal_snapshot2": "test_2_y", "xtal_snapshot3": "test_3_y", - "synchrotron_mode": None, + "synchrotron_mode": "test", "undulator_gap1": 1.0, "starttime": EXPECTED_START_TIME, "filetemplate": "file_name_0_master.h5", @@ -371,9 +459,9 @@ def test_update_deposition( mx_acq.get_dc_position_params(), { "id": TEST_DATA_COLLECTION_IDS[0], - "pos_x": dummy_params_3d.hyperion_params.ispyb_params.position[0], - "pos_y": dummy_params_3d.hyperion_params.ispyb_params.position[1], - "pos_z": dummy_params_3d.hyperion_params.ispyb_params.position[2], + "pos_x": 0, + "pos_y": 0, + "pos_z": 0, }, ) @@ -382,18 +470,14 @@ def test_update_deposition( mx_acq.get_dc_grid_params(), { "parentid": TEST_DATA_COLLECTION_IDS[0], - "dxinmm": dummy_params_3d.experiment_params.x_step_size, - "dyinmm": dummy_params_3d.experiment_params.y_step_size, - "stepsx": dummy_params_3d.experiment_params.x_steps, - "stepsy": dummy_params_3d.experiment_params.y_steps, - "micronsperpixelx": dummy_params_3d.hyperion_params.ispyb_params.microns_per_pixel_x, - "micronsperpixely": dummy_params_3d.hyperion_params.ispyb_params.microns_per_pixel_y, - "snapshotoffsetxpixel": dummy_params_3d.hyperion_params.ispyb_params.upper_left[ - 0 - ], - "snapshotoffsetypixel": dummy_params_3d.hyperion_params.ispyb_params.upper_left[ - 1 - ], + "dxinmm": 0.1, + "dyinmm": 0.1, + "stepsx": 40, + "stepsy": 20, + "micronsperpixelx": 1.25, + "micronsperpixely": 1.25, + "snapshotoffsetxpixel": 50, + "snapshotoffsetypixel": 100, "orientation": "horizontal", "snaked": True, }, @@ -439,7 +523,7 @@ def test_update_deposition( "xtal_snapshot1": "test_1_z", "xtal_snapshot2": "test_2_z", "xtal_snapshot3": "test_3_z", - "synchrotron_mode": None, + "synchrotron_mode": "test", "undulator_gap1": 1.0, "starttime": EXPECTED_START_TIME, "filetemplate": "file_name_1_master.h5", @@ -452,9 +536,9 @@ def test_update_deposition( mx_acq.get_dc_position_params(), { "id": TEST_DATA_COLLECTION_IDS[1], - "pos_x": dummy_params_3d.hyperion_params.ispyb_params.position[0], - "pos_y": dummy_params_3d.hyperion_params.ispyb_params.position[1], - "pos_z": dummy_params_3d.hyperion_params.ispyb_params.position[2], + "pos_x": 0, + "pos_y": 0, + "pos_z": 0, }, ) @@ -463,18 +547,14 @@ def test_update_deposition( mx_acq.get_dc_grid_params(), { "parentid": TEST_DATA_COLLECTION_IDS[1], - "dxinmm": dummy_params_3d.experiment_params.x_step_size, - "dyinmm": dummy_params_3d.experiment_params.z_step_size, - "stepsx": dummy_params_3d.experiment_params.x_steps, - "stepsy": dummy_params_3d.experiment_params.z_steps, - "micronsperpixelx": dummy_params_3d.hyperion_params.ispyb_params.microns_per_pixel_x, - "micronsperpixely": dummy_params_3d.hyperion_params.ispyb_params.microns_per_pixel_y, - "snapshotoffsetxpixel": dummy_params_3d.hyperion_params.ispyb_params.upper_left[ - 0 - ], - "snapshotoffsetypixel": dummy_params_3d.hyperion_params.ispyb_params.upper_left[ - 2 - ], + "dxinmm": 0.1, + "dyinmm": 0.2, + "stepsx": 40, + "stepsy": 10, + "micronsperpixelx": 1.25, + "micronsperpixely": 1.25, + "snapshotoffsetxpixel": 50, + "snapshotoffsetypixel": 120, "orientation": "horizontal", "snaked": True, }, @@ -492,7 +572,6 @@ def test_end_deposition_happy_path( get_current_time, mock_ispyb_conn, dummy_3d_gridscan_ispyb, - dummy_params_3d, dummy_collection_group_info, scan_data_info_for_begin, scan_data_infos_for_update, @@ -544,3 +623,164 @@ def test_end_deposition_happy_path( "runstatus": "DataCollection Successful", }, ) + + +def test_param_keys( + mock_ispyb_conn, + dummy_2d_gridscan_ispyb, + dummy_collection_group_info, + scan_data_info_for_begin, + scan_xy_data_info_for_update, +): + ispyb_ids = dummy_2d_gridscan_ispyb.begin_deposition( + dummy_collection_group_info, scan_data_info_for_begin + ) + assert dummy_2d_gridscan_ispyb.update_deposition( + ispyb_ids, dummy_collection_group_info, [scan_xy_data_info_for_update] + ) == IspybIds( + data_collection_ids=(TEST_DATA_COLLECTION_IDS[0],), + data_collection_group_id=TEST_DATA_COLLECTION_GROUP_ID, + grid_ids=(TEST_GRID_INFO_IDS[0],), + ) + + +def _test_when_grid_scan_stored_then_data_present_in_upserts( + ispyb_conn, + dummy_ispyb, + test_function, + dummy_collection_group_info, + scan_data_info_for_begin, + scan_data_info_for_update, + test_group=False, +): + setup_mock_return_values(ispyb_conn) + ispyb_ids = dummy_ispyb.begin_deposition( + dummy_collection_group_info, scan_data_info_for_begin + ) + dummy_ispyb.update_deposition( + ispyb_ids, dummy_collection_group_info, [scan_data_info_for_update] + ) + + mx_acquisition = ispyb_conn.return_value.__enter__.return_value.mx_acquisition + + upsert_data_collection_arg_list = ( + mx_acquisition.upsert_data_collection.call_args_list[1][0] + ) + actual = upsert_data_collection_arg_list[0] + assert test_function(MXAcquisition.get_data_collection_params(), actual) + + if test_group: + upsert_data_collection_group_arg_list = ( + mx_acquisition.upsert_data_collection_group.call_args_list[1][0] + ) + actual = upsert_data_collection_group_arg_list[0] + assert test_function(MXAcquisition.get_data_collection_group_params(), actual) + + +@patch("ispyb.open", autospec=True) +def test_given_sampleid_of_none_when_grid_scan_stored_then_sample_id_not_set( + ispyb_conn, + dummy_2d_gridscan_ispyb, + dummy_collection_group_info, + scan_data_info_for_begin, + scan_xy_data_info_for_update, +): + dummy_collection_group_info.sample_id = None + scan_data_info_for_begin.data_collection_info.sample_id = None + scan_xy_data_info_for_update.data_collection_info.sample_id = None + + def test_sample_id(default_params, actual): + sampleid_idx = list(default_params).index("sampleid") + return actual[sampleid_idx] == default_params["sampleid"] + + _test_when_grid_scan_stored_then_data_present_in_upserts( + ispyb_conn, + dummy_2d_gridscan_ispyb, + test_sample_id, + dummy_collection_group_info, + scan_data_info_for_begin, + scan_xy_data_info_for_update, + True, + ) + + +@patch("ispyb.open", autospec=True) +def test_given_real_sampleid_when_grid_scan_stored_then_sample_id_set( + ispyb_conn, + dummy_2d_gridscan_ispyb: StoreInIspyb, + dummy_collection_group_info, + scan_data_info_for_begin, + scan_xy_data_info_for_update, +): + expected_sample_id = "0001" + + def test_sample_id(default_params, actual): + sampleid_idx = list(default_params).index("sampleid") + return actual[sampleid_idx] == expected_sample_id + + _test_when_grid_scan_stored_then_data_present_in_upserts( + ispyb_conn, + dummy_2d_gridscan_ispyb, + test_sample_id, + dummy_collection_group_info, + scan_data_info_for_begin, + scan_xy_data_info_for_update, + True, + ) + + +def test_fail_result_run_results_in_bad_run_status( + mock_ispyb_conn: MagicMock, + dummy_2d_gridscan_ispyb: StoreInIspyb, + dummy_collection_group_info, + scan_data_info_for_begin, + scan_xy_data_info_for_update, +): + mock_ispyb_conn = mock_ispyb_conn + mock_mx_aquisition = ( + mock_ispyb_conn.return_value.__enter__.return_value.mx_acquisition + ) + mock_upsert_data_collection = mock_mx_aquisition.upsert_data_collection + + ispyb_ids = dummy_2d_gridscan_ispyb.begin_deposition( + dummy_collection_group_info, scan_data_info_for_begin + ) + ispyb_ids = dummy_2d_gridscan_ispyb.update_deposition( + ispyb_ids, dummy_collection_group_info, [scan_xy_data_info_for_update] + ) + dummy_2d_gridscan_ispyb.end_deposition(ispyb_ids, "fail", "test specifies failure") + + mock_upsert_data_collection_calls = mock_upsert_data_collection.call_args_list + end_deposition_upsert_args = mock_upsert_data_collection_calls[2][0] + upserted_param_value_list = end_deposition_upsert_args[0] + assert "DataCollection Unsuccessful" in upserted_param_value_list + assert "DataCollection Successful" not in upserted_param_value_list + + +def test_no_exception_during_run_results_in_good_run_status( + mock_ispyb_conn: MagicMock, + dummy_2d_gridscan_ispyb: StoreInIspyb, + dummy_collection_group_info, + scan_data_info_for_begin, + scan_xy_data_info_for_update, +): + mock_ispyb_conn = mock_ispyb_conn + setup_mock_return_values(mock_ispyb_conn) + mock_mx_aquisition = ( + mock_ispyb_conn.return_value.__enter__.return_value.mx_acquisition + ) + mock_upsert_data_collection = mock_mx_aquisition.upsert_data_collection + + ispyb_ids = dummy_2d_gridscan_ispyb.begin_deposition( + dummy_collection_group_info, scan_data_info_for_begin + ) + ispyb_ids = dummy_2d_gridscan_ispyb.update_deposition( + ispyb_ids, dummy_collection_group_info, [scan_xy_data_info_for_update] + ) + dummy_2d_gridscan_ispyb.end_deposition(ispyb_ids, "success", "") + + mock_upsert_data_collection_calls = mock_upsert_data_collection.call_args_list + end_deposition_upsert_args = mock_upsert_data_collection_calls[2][0] + upserted_param_value_list = end_deposition_upsert_args[0] + assert "DataCollection Unsuccessful" not in upserted_param_value_list + assert "DataCollection Successful" in upserted_param_value_list diff --git a/tests/unit_tests/external_interaction/ispyb/test_rotation_ispyb_store.py b/tests/unit_tests/external_interaction/ispyb/test_rotation_ispyb_store.py index 50b13c6f4..28d887322 100644 --- a/tests/unit_tests/external_interaction/ispyb/test_rotation_ispyb_store.py +++ b/tests/unit_tests/external_interaction/ispyb/test_rotation_ispyb_store.py @@ -1,18 +1,14 @@ -from unittest.mock import MagicMock, mock_open, patch +from unittest.mock import MagicMock, patch import pytest -from mockito import mock -from hyperion.external_interaction.callbacks.common.ispyb_mapping import ( - populate_data_collection_group, - populate_data_collection_position_info, - populate_remaining_data_collection_info, +from hyperion.external_interaction.ispyb.data_model import ( + DataCollectionGroupInfo, + DataCollectionInfo, + DataCollectionPositionInfo, + ExperimentType, + ScanDataInfo, ) -from hyperion.external_interaction.callbacks.rotation.ispyb_mapping import ( - construct_comment_for_rotation_scan, - populate_data_collection_info_for_rotation, -) -from hyperion.external_interaction.ispyb.data_model import ExperimentType, ScanDataInfo from hyperion.external_interaction.ispyb.ispyb_store import ( IspybIds, StoreInIspyb, @@ -41,8 +37,6 @@ "axisend": 180, "focal_spot_size_at_samplex": 1.0, "focal_spot_size_at_sampley": 1.0, - "slitgap_vertical": 1, - "slitgap_horizontal": 1, "beamsize_at_samplex": 1, "beamsize_at_sampley": 1, "transmission": 100.0, @@ -55,7 +49,6 @@ "imgsuffix": "h5", "n_passes": 1, "overlap": 0, - "flux": 10.0, "omegastart": 0, "start_image_number": 1, "resolution": 1.0, # deferred @@ -74,11 +67,11 @@ @pytest.fixture -def dummy_rotation_data_collection_group_info(dummy_rotation_params): - return populate_data_collection_group( - "SAD", - dummy_rotation_params.hyperion_params.detector_params, - dummy_rotation_params.hyperion_params.ispyb_params, +def dummy_rotation_data_collection_group_info(): + return DataCollectionGroupInfo( + visit_string="cm31105-4", + experiment_type="SAD", + sample_id="0001", ) @@ -87,36 +80,109 @@ def dummy_rotation_data_collection_group_info(dummy_rotation_params): "hyperion.external_interaction.callbacks.common.ispyb_mapping.get_current_time_string", new=MagicMock(return_value=EXPECTED_START_TIME), ) -def scan_data_info_for_begin(dummy_rotation_params): - scan_data_info = ScanDataInfo( - data_collection_info=populate_remaining_data_collection_info( - construct_comment_for_rotation_scan, - None, - populate_data_collection_info_for_rotation( - dummy_rotation_params.hyperion_params.ispyb_params, - dummy_rotation_params.hyperion_params.detector_params, - dummy_rotation_params, - ), - dummy_rotation_params.hyperion_params.detector_params, - dummy_rotation_params.hyperion_params.ispyb_params, - ) - ) - return scan_data_info +def scan_data_info_for_begin(): + return ScanDataInfo( + data_collection_info=DataCollectionInfo( + omega_start=0.0, + data_collection_number=0, + xtal_snapshot1="test_1_y", + xtal_snapshot2="test_2_y", + xtal_snapshot3="test_3_y", + n_images=1800, + axis_range=0.1, + axis_end=180.0, + kappa_start=0.0, + parent_id=None, + visit_string="cm31105-4", + sample_id="0001", + detector_id=78, + axis_start=0.0, + focal_spot_size_at_samplex=1.0, + focal_spot_size_at_sampley=1.0, + beamsize_at_samplex=1.0, + beamsize_at_sampley=1.0, + transmission=100.0, + comments="Hyperion rotation scan", + detector_distance=100.0, + exp_time=0.1, + imgdir="/tmp/", + file_template="file_name_0_master.h5", + imgprefix="file_name", + imgsuffix="h5", + n_passes=1, + overlap=0, + start_image_number=1, + resolution=1.0, + wavelength=123.98419840550369, + xbeam=150.0, + ybeam=160.0, + synchrotron_mode=None, + undulator_gap1=None, + start_time="2024-02-08 14:03:59", + ), + data_collection_id=None, + data_collection_position_info=None, + data_collection_grid_info=None, + ) @pytest.fixture -def scan_data_info_for_update(scan_data_info_for_begin, dummy_rotation_params): - scan_data_info_for_begin.data_collection_position_info = ( - populate_data_collection_position_info( - dummy_rotation_params.hyperion_params.ispyb_params - ) +def scan_data_info_for_update(scan_data_info_for_begin): + return ScanDataInfo( + data_collection_info=DataCollectionInfo( + omega_start=0.0, + data_collection_number=0, + xtal_snapshot1="test_1_y", + xtal_snapshot2="test_2_y", + xtal_snapshot3="test_3_y", + n_images=1800, + axis_range=0.1, + axis_end=180.0, + kappa_start=0.0, + parent_id=None, + visit_string="cm31105-4", + sample_id="0001", + detector_id=78, + axis_start=0.0, + focal_spot_size_at_samplex=1.0, + focal_spot_size_at_sampley=1.0, + slitgap_vertical=1.0, + slitgap_horizontal=1.0, + beamsize_at_samplex=1.0, + beamsize_at_sampley=1.0, + transmission=100.0, + comments="Hyperion rotation scan", + detector_distance=100.0, + exp_time=0.1, + imgdir="/tmp/", + file_template="file_name_0_master.h5", + imgprefix="file_name", + imgsuffix="h5", + n_passes=1, + overlap=0, + flux=10.0, + start_image_number=1, + resolution=1.0, + wavelength=123.98419840550369, + xbeam=150.0, + ybeam=160.0, + synchrotron_mode="test", + undulator_gap1=None, + start_time="2024-02-08 14:03:59", + ), + data_collection_id=11, + data_collection_position_info=DataCollectionPositionInfo( + pos_x=10.0, pos_y=20.0, pos_z=30.0 + ), + data_collection_grid_info=None, ) - return scan_data_info_for_begin @pytest.fixture -def dummy_rotation_ispyb_with_experiment_type(dummy_rotation_params): - store_in_ispyb = StoreInIspyb(CONST.SIM.ISPYB_CONFIG, "Characterization") +def dummy_rotation_ispyb_with_experiment_type(): + store_in_ispyb = StoreInIspyb( + CONST.SIM.ISPYB_CONFIG, ExperimentType.CHARACTERIZATION + ) return store_in_ispyb @@ -127,16 +193,23 @@ def dummy_rotation_ispyb_with_experiment_type(dummy_rotation_params): def test_begin_deposition( mock_ispyb_conn, dummy_rotation_ispyb, - dummy_rotation_params, dummy_rotation_data_collection_group_info, scan_data_info_for_begin, ): + assert scan_data_info_for_begin.data_collection_info.parent_id is None + assert dummy_rotation_ispyb.begin_deposition( dummy_rotation_data_collection_group_info, scan_data_info_for_begin ) == IspybIds( data_collection_ids=(TEST_DATA_COLLECTION_IDS[0],), data_collection_group_id=TEST_DATA_COLLECTION_GROUP_ID, ) + + assert ( + scan_data_info_for_begin.data_collection_info.parent_id + == TEST_DATA_COLLECTION_GROUP_ID + ) + mx_acq = mx_acquisition_from_conn(mock_ispyb_conn) assert_upsert_call_with( mx_acq.upsert_data_collection_group.mock_calls[0], @@ -145,7 +218,6 @@ def test_begin_deposition( "parentid": TEST_SESSION_ID, "experimenttype": "SAD", "sampleid": TEST_SAMPLE_ID, - "sample_barcode": TEST_BARCODE, # deferred }, ) assert_upsert_call_with( @@ -159,9 +231,8 @@ def test_begin_deposition( "hyperion.external_interaction.callbacks.common.ispyb_mapping.get_current_time_string", new=MagicMock(return_value=EXPECTED_START_TIME), ) -def test_begin_deposition_with_group_id_doesnt_insert( +def test_begin_deposition_with_group_id_updates_but_doesnt_insert( mock_ispyb_conn, - dummy_rotation_params, dummy_rotation_data_collection_group_info, scan_data_info_for_begin, ): @@ -169,6 +240,7 @@ def test_begin_deposition_with_group_id_doesnt_insert( scan_data_info_for_begin.data_collection_info.parent_id = ( TEST_DATA_COLLECTION_GROUP_ID ) + assert dummy_rotation_ispyb.begin_deposition( dummy_rotation_data_collection_group_info, scan_data_info_for_begin ) == IspybIds( @@ -176,7 +248,21 @@ def test_begin_deposition_with_group_id_doesnt_insert( data_collection_group_id=TEST_DATA_COLLECTION_GROUP_ID, ) mx_acq = mx_acquisition_from_conn(mock_ispyb_conn) - mx_acq.upsert_data_collection_group.assert_not_called() + assert_upsert_call_with( + mx_acq.upsert_data_collection_group.mock_calls[0], + mx_acq.get_data_collection_group_params(), + { + "id": TEST_DATA_COLLECTION_GROUP_ID, + "parentid": TEST_SESSION_ID, + "experimenttype": "SAD", + "sampleid": TEST_SAMPLE_ID, + }, + ) + assert ( + scan_data_info_for_begin.data_collection_info.parent_id + == TEST_DATA_COLLECTION_GROUP_ID + ) + assert_upsert_call_with( mx_acq.upsert_data_collection.mock_calls[0], mx_acq.get_data_collection_params(), @@ -191,7 +277,6 @@ def test_begin_deposition_with_group_id_doesnt_insert( def test_begin_deposition_with_alternate_experiment_type( mock_ispyb_conn, dummy_rotation_ispyb_with_experiment_type, - dummy_rotation_params, dummy_rotation_data_collection_group_info, scan_data_info_for_begin, ): @@ -211,7 +296,6 @@ def test_begin_deposition_with_alternate_experiment_type( "parentid": TEST_SESSION_ID, "experimenttype": "Characterization", "sampleid": TEST_SAMPLE_ID, - "sample_barcode": TEST_BARCODE, # deferred }, ) @@ -223,7 +307,6 @@ def test_begin_deposition_with_alternate_experiment_type( def test_update_deposition( mock_ispyb_conn, dummy_rotation_ispyb, - dummy_rotation_params, dummy_rotation_data_collection_group_info, scan_data_info_for_begin, scan_data_info_for_update, @@ -235,6 +318,12 @@ def test_update_deposition( mx_acq.upsert_data_collection_group.reset_mock() mx_acq.upsert_data_collection.reset_mock() + scan_data_info_for_update.data_collection_info.parent_id = ( + ispyb_ids.data_collection_group_id + ) + scan_data_info_for_update.data_collection_id = ispyb_ids.data_collection_ids[0] + dummy_rotation_data_collection_group_info.sample_barcode = TEST_BARCODE + assert dummy_rotation_ispyb.update_deposition( ispyb_ids, dummy_rotation_data_collection_group_info, @@ -260,6 +349,10 @@ def test_update_deposition( EXPECTED_DATA_COLLECTION | { "id": TEST_DATA_COLLECTION_IDS[0], + "synchrotron_mode": "test", + "slitgap_vertical": 1, + "slitgap_horizontal": 1, + "flux": 10, }, ) @@ -268,9 +361,9 @@ def test_update_deposition( mx_acq.get_dc_position_params(), { "id": TEST_DATA_COLLECTION_IDS[0], - "pos_x": dummy_rotation_params.hyperion_params.ispyb_params.position[0], - "pos_y": dummy_rotation_params.hyperion_params.ispyb_params.position[1], - "pos_z": dummy_rotation_params.hyperion_params.ispyb_params.position[2], + "pos_x": 10, + "pos_y": 20, + "pos_z": 30, }, ) @@ -281,7 +374,6 @@ def test_update_deposition( ) def test_update_deposition_with_group_id_updates( mock_ispyb_conn, - dummy_rotation_params, dummy_rotation_data_collection_group_info, scan_data_info_for_begin, scan_data_info_for_update, @@ -297,6 +389,11 @@ def test_update_deposition_with_group_id_updates( mx_acq.upsert_data_collection_group.reset_mock() mx_acq.upsert_data_collection.reset_mock() + scan_data_info_for_update.data_collection_info.parent_id = ( + ispyb_ids.data_collection_group_id + ) + scan_data_info_for_update.data_collection_id = ispyb_ids.data_collection_ids[0] + dummy_rotation_data_collection_group_info.sample_barcode = TEST_BARCODE assert dummy_rotation_ispyb.update_deposition( ispyb_ids, dummy_rotation_data_collection_group_info, @@ -322,6 +419,10 @@ def test_update_deposition_with_group_id_updates( EXPECTED_DATA_COLLECTION | { "id": TEST_DATA_COLLECTION_IDS[0], + "synchrotron_mode": "test", + "slitgap_vertical": 1, + "slitgap_horizontal": 1, + "flux": 10, }, ) @@ -330,9 +431,9 @@ def test_update_deposition_with_group_id_updates( mx_acq.get_dc_position_params(), { "id": TEST_DATA_COLLECTION_IDS[0], - "pos_x": dummy_rotation_params.hyperion_params.ispyb_params.position[0], - "pos_y": dummy_rotation_params.hyperion_params.ispyb_params.position[1], - "pos_z": dummy_rotation_params.hyperion_params.ispyb_params.position[2], + "pos_x": 10, + "pos_y": 20, + "pos_z": 30, }, ) @@ -348,7 +449,6 @@ def test_end_deposition_happy_path( get_current_time, mock_ispyb_conn, dummy_rotation_ispyb, - dummy_rotation_params, dummy_rotation_data_collection_group_info, scan_data_info_for_begin, scan_data_info_for_update, @@ -356,6 +456,10 @@ def test_end_deposition_happy_path( ispyb_ids = dummy_rotation_ispyb.begin_deposition( dummy_rotation_data_collection_group_info, scan_data_info_for_begin ) + scan_data_info_for_update.data_collection_info.parent_id = ( + ispyb_ids.data_collection_group_id + ) + scan_data_info_for_update.data_collection_id = ispyb_ids.data_collection_ids[0] ispyb_ids = dummy_rotation_ispyb.update_deposition( ispyb_ids, dummy_rotation_data_collection_group_info, @@ -400,38 +504,34 @@ def test_store_rotation_scan_failures( dummy_rotation_ispyb.end_deposition(ispyb_ids, "", "") -def test_populate_data_collection_info_for_rotation_checks_snapshots( - dummy_rotation_params, dummy_rotation_data_collection_group_info -): - with patch("hyperion.log.ISPYB_LOGGER.warning", autospec=True) as warning: - dummy_rotation_params.hyperion_params.ispyb_params.xtal_snapshots_omega_start = ( - None - ) - populate_data_collection_info_for_rotation( - dummy_rotation_params.hyperion_params.ispyb_params, - dummy_rotation_params.hyperion_params.detector_params, - dummy_rotation_params, - ) - warning.assert_called_once_with("No xtal snapshot paths sent to ISPyB!") - - @pytest.mark.parametrize("dcgid", [2, 45, 61, 88, 13, 25]) -@patch("ispyb.open", new_callable=mock_open) def test_store_rotation_scan_uses_supplied_dcgid( - ispyb_conn, + mock_ispyb_conn, dcgid, dummy_rotation_data_collection_group_info, scan_data_info_for_begin, scan_data_info_for_update, ): - ispyb_conn.return_value.mx_acquisition = MagicMock() - ispyb_conn.return_value.core = mock() + mock_ispyb_conn.return_value.mx_acquisition.upsert_data_collection_group.return_value = ( + dcgid + ) store_in_ispyb = StoreInIspyb(CONST.SIM.ISPYB_CONFIG, ExperimentType.ROTATION) scan_data_info_for_begin.data_collection_info.parent_id = dcgid ispyb_ids = store_in_ispyb.begin_deposition( dummy_rotation_data_collection_group_info, scan_data_info_for_begin ) assert ispyb_ids.data_collection_group_id == dcgid + mx = mx_acquisition_from_conn(mock_ispyb_conn) + assert_upsert_call_with( + mx.upsert_data_collection_group.mock_calls[0], + mx.get_data_collection_group_params(), + { + "id": dcgid, + "parentid": TEST_SESSION_ID, + "experimenttype": "SAD", + "sampleid": TEST_SAMPLE_ID, + }, + ) assert ( store_in_ispyb.update_deposition( ispyb_ids, diff --git a/tests/unit_tests/parameters/test_internal_parameters.py b/tests/unit_tests/parameters/test_internal_parameters.py index 0c0d9f9e0..fcc81efde 100644 --- a/tests/unit_tests/parameters/test_internal_parameters.py +++ b/tests/unit_tests/parameters/test_internal_parameters.py @@ -76,8 +76,9 @@ def test_ispyb_param_wavelength(): from hyperion.utils.utils import convert_eV_to_angstrom ispyb_params = GridscanIspybParams(**GRIDSCAN_ISPYB_PARAM_DEFAULTS) + ispyb_params.current_energy_ev = 12700 assert ispyb_params.wavelength_angstroms == pytest.approx( - convert_eV_to_angstrom(GRIDSCAN_ISPYB_PARAM_DEFAULTS["current_energy_ev"]) + convert_eV_to_angstrom(12700) )