diff --git a/.ci/bl-specific.sh b/.ci/bl-specific.sh index cf87896..ea71d41 100644 --- a/.ci/bl-specific.sh +++ b/.ci/bl-specific.sh @@ -2,4 +2,14 @@ # cp -v <...> ~/.ipython/profile_${TEST_PROFILE}/ -conda install -y -c ${CONDA_CHANNEL_NAME} 28-id-1-pdf-collection +conda install -y -c ${CONDA_CHANNEL_NAME} xpdacq + +mkdir -v -p ~/.config/acq/ +mkdir -v -p ~/user_data/config_base/yml/ +sudo mkdir -v -p /mnt/data/bnl/xpdacq_special/data/xpdConfig/ +sudo chown -Rv $USER: /mnt/data/bnl/xpdacq_special/data/xpdConfig/ + +cp -v .ci/pdf.yml ~/.config/acq/ +cp -v .ci/glbl.yml ~/user_data/config_base/yml/ +cp -v .ci/xpd_beamline_config.yml /mnt/data/bnl/xpdacq_special/data/xpdConfig/ + diff --git a/.ci/glbl.yml b/.ci/glbl.yml new file mode 100644 index 0000000..5db2bb9 --- /dev/null +++ b/.ci/glbl.yml @@ -0,0 +1,158 @@ +_dark_dict_list: +- acq_time: 5 + exposure: 5 + timestamp: 1614460063.2800422 + uid: 37d192fb-df2c-44a9-b41b-2a05c9751247 +- acq_time: 5 + exposure: 5 + timestamp: 1614460242.3911414 + uid: cf374ae7-2ed5-4e5d-9563-f4841a3759e9 +- acq_time: 5 + exposure: 5 + timestamp: 1614460464.3035522 + uid: 5fc90d2e-ef96-4ca7-9207-ae72121923d0 +- acq_time: 5 + exposure: !!python/object/apply:numpy.core.multiarray.scalar + - &id001 !!python/object/apply:numpy.dtype + args: + - f8 + - false + - true + state: !!python/tuple + - 3 + - < + - null + - null + - null + - -1 + - -1 + - 0 + - !!binary | + AAAAAAAAFEA= + timestamp: 1614460566.1485345 + uid: 12c2f3ca-b459-4180-a12e-2b2d4b6d5aa8 +- acq_time: 5 + exposure: !!python/object/apply:numpy.core.multiarray.scalar + - *id001 + - !!binary | + AAAAAAAAFEA= + timestamp: 1614464054.0055833 + uid: 5bf51ebe-6439-4815-a9bb-0c824fd3afe8 +- acq_time: 5 + exposure: !!python/object/apply:numpy.core.multiarray.scalar + - *id001 + - !!binary | + AAAAAAAAFEA= + timestamp: 1614464263.5876548 + uid: 25d95820-b20f-4d37-901d-6c96c86462a1 +- acq_time: 5 + exposure: !!python/object/apply:numpy.core.multiarray.scalar + - *id001 + - !!binary | + AAAAAAAAFEA= + timestamp: 1614464433.3411603 + uid: bb969c1c-2e41-4914-91db-3440b576c752 +- acq_time: 5 + exposure: !!python/object/apply:numpy.core.multiarray.scalar + - *id001 + - !!binary | + AAAAAAAAFEA= + timestamp: 1614464443.6444564 + uid: 319ccd2f-7483-459c-96ac-e308b150006e +_exclude_dir: +- /mnt/data/bnl/xpdacq_special/data/user_data +- /mnt/data/bnl/xpdacq_special/data/xpdConfig +- /mnt/data/bnl/xpdacq_special/data/user_data/config_base/yml +_export_tar_dir: +- /mnt/data/bnl/xpdacq_special/data/user_data/config_base +- /mnt/data/bnl/xpdacq_special/data/user_data/userScripts +all_folders: &id002 +- /mnt/data/bnl/xpdacq_special/data/user_data +- /mnt/data/bnl/xpdacq_special/data/xpdConfig +- /mnt/data/bnl/xpdacq_special/data/user_data/config_base/yml +- /mnt/data/bnl/xpdacq_special/data/user_data/config_base +- /mnt/data/bnl/xpdacq_special/data/user_data/config_base/yml/samples +- /mnt/data/bnl/xpdacq_special/data/user_data/config_base/yml/scanplans +- /mnt/data/bnl/xpdacq_special/data/user_data/tiff_base +- /mnt/data/bnl/xpdacq_special/data/user_data/userScripts +- /mnt/data/bnl/xpdacq_special/data/user_data/Import +- /mnt/data/bnl/xpdacq_special/data/user_data/userAnalysis +allfolders: *id002 +archive_base_dir: /mnt/data/bnl/xpdacq_special/archive/.userbeamtimearchive +archive_base_dir_name: .userbeamtimearchive +archive_dir: /mnt/data/bnl/xpdacq_special/archive/.userbeamtimearchive/2021 +archive_root_dir: /mnt/data/bnl/xpdacq_special/archive +auto_dark: true +auto_load_calib: true +base: /mnt/data/bnl/xpdacq_special/data +base_dir: /mnt/data/bnl/xpdacq_special/data +beamline_host_name: +- jupiter +beamline_id: 28-ID-1 +blconfig_dir: /mnt/data/bnl/xpdacq_special/data/xpdConfig +blconfig_dir_name: xpdConfig +blconfig_name: xpd_beamline_config.yml +blconfig_path: /mnt/data/bnl/xpdacq_special/data/xpdConfig/xpd_beamline_config.yml +bt_dir: /mnt/data/bnl/xpdacq_special/data/user_data/config_base/yml +calib_config_name: xpdAcq_calib_info.yml +config_base: /mnt/data/bnl/xpdacq_special/data/user_data/config_base +dark_field_key: sc_dk_field_uid +dark_window: 0.1 +det_image_field: &id003 +- pe1c_image +- pe2c_image +- dexela_image +- dexela +diffraction_dets: +- pe1c +- pe1 +- pe2c +- pe2 +- dexela_image +- dexela +dk_window: 0.1 +exp_broker_name: pdf +exp_hash_uid: f4be92ca-79c1-49ae-9bcd-59c7a8185aa3 +facility: NSLS-II +frame_acq_time: 0.1 +frame_acquire_time: 0.1 +glbl_yaml_name: glbl.yml +glbl_yaml_path: /mnt/data/bnl/xpdacq_special/data/user_data/config_base/yml/glbl.yml +group: XPD +home: /mnt/data/bnl/xpdacq_special/data/user_data +home_dir: /mnt/data/bnl/xpdacq_special/data/user_data +home_dir_name: user_data +image_field: pe1c_image +image_fields: *id003 +import_dir: /mnt/data/bnl/xpdacq_special/data/user_data/Import +inbound_proxy_address: xf28id1-ca1:5577 +is_simulation: false +mask_kwargs: + alpha: 3.0 + bs_width: 13 + edge: 30 + lower_thresh: 0.0 + tri_offset: 13 + v_asym: 0 +name: glbl +outbound_proxy_address: xf28id1-ca1:5578 +owner: PDF +radiogram_dets: +- blackfly_det +radiograph_names: +- blackfly_det +sample_dir: /mnt/data/bnl/xpdacq_special/data/user_data/config_base/yml/samples +scanplan_dir: /mnt/data/bnl/xpdacq_special/data/user_data/config_base/yml/scanplans +shutter_conf: + close: 0 + open: 1 +shutter_control: true +shutter_sleep: 0 +simulation: false +tiff_base: /mnt/data/bnl/xpdacq_special/data/user_data/tiff_base +user_backup_dir_name: '2021' +userscript_dir: /mnt/data/bnl/xpdacq_special/data/user_data/userScripts +usrAnalysis_dir: /mnt/data/bnl/xpdacq_special/data/user_data/userAnalysis +usrScript_dir: /mnt/data/bnl/xpdacq_special/data/user_data/userScripts +xpdconfig: /mnt/data/bnl/xpdacq_special/data/xpdConfig +yaml_dir: /mnt/data/bnl/xpdacq_special/data/user_data/config_base/yml diff --git a/.ci/pdf.yml b/.ci/pdf.yml new file mode 100644 index 0000000..68f84b0 --- /dev/null +++ b/.ci/pdf.yml @@ -0,0 +1,35 @@ +archive_base_dir_name: .userbeamtimearchive +archive_root_dir: /mnt/data/bnl/xpdacq_special/archive +base_dir: /mnt/data/bnl/xpdacq_special/data +beamline_host_name: [jupiter] +beamline_id: 28-ID-1 +blconfig_dir_name: xpdConfig +blconfig_name: xpd_beamline_config.yml +calib_config_name: xpdAcq_calib_info.yml +dark_window: 0.1 +facility: NSLS-II +frame_acquire_time: 0.1 +glbl_yaml_name: glbl.yml +group: XPD +home_dir_name: user_data +image_field: pe1c_image +owner: PDF +simulation: false +shutter_conf: {close: 0, open: 1} +mask_kwargs: + edge: 30 + lower_thresh: 0.0 + bs_width: 13 + tri_offset: 13 + v_asym: 0 + alpha: 3.0 +dark_field_key : sc_dk_field_uid +det_image_field : pe1_image +exp_broker_name: pdf +outbound_proxy_address: 'xf28id1-ca1:5578' +inbound_proxy_address: 'xf28id1-ca1:5577' +shutter_sleep: 0 +diffraction_dets: ['pe1c', 'pe1', 'pe2c', 'pe2','dexela_image', 'dexela'] +radiograph_names : ['blackfly_det'] +radiogram_dets: ['blackfly_det'] +image_fields: ['pe1c_image', 'pe2c_image', 'dexela_image', 'dexela'] diff --git a/.ci/xpd_beamline_config.yml b/.ci/xpd_beamline_config.yml new file mode 100644 index 0000000..f3fa7ed --- /dev/null +++ b/.ci/xpd_beamline_config.yml @@ -0,0 +1,4 @@ +Verification time: '2021-01-09 17:45:59' +Verified by: AUTO VERIFIED IN TEST +hadtoedit: false +is_pytest: false diff --git a/azure-pipelines.yml b/azure-pipelines.yml index d04fcca..e8faeec 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -3,10 +3,10 @@ resources: - repository: templates type: github name: NSLS-II/profile-collection-ci - # ref: refs/heads/ # for future testings on a branch of NSLS-II/profile-collection-ci + ref: refs/heads/main endpoint: github jobs: -- template: azure-linux.yml@templates # Template reference - parameters: - beamline_acronym: PDF + - template: collection-2021-1.0.yml@templates + parameters: + beamline_acronym: PDF diff --git a/startup/00-base.py b/startup/00-base.py index 52bad2f..20ae94b 100644 --- a/startup/00-base.py +++ b/startup/00-base.py @@ -1,3 +1,47 @@ +try: + ############################################################################### + # TODO: remove this block once https://github.com/bluesky/ophyd/pull/959 is + # merged/released. + from datetime import datetime + from ophyd.signal import EpicsSignalBase, EpicsSignal, DEFAULT_CONNECTION_TIMEOUT + + def print_now(): + return datetime.strftime(datetime.now(), '%Y-%m-%d %H:%M:%S.%f') + + def wait_for_connection_base(self, timeout=DEFAULT_CONNECTION_TIMEOUT): + '''Wait for the underlying signals to initialize or connect''' + if timeout is DEFAULT_CONNECTION_TIMEOUT: + timeout = self.connection_timeout + # print(f'{print_now()}: waiting for {self.name} to connect within {timeout:.4f} s...') + start = time.time() + try: + self._ensure_connected(self._read_pv, timeout=timeout) + # print(f'{print_now()}: waited for {self.name} to connect for {time.time() - start:.4f} s.') + except TimeoutError: + if self._destroyed: + raise DestroyedError('Signal has been destroyed') + raise + + def wait_for_connection(self, timeout=DEFAULT_CONNECTION_TIMEOUT): + '''Wait for the underlying signals to initialize or connect''' + if timeout is DEFAULT_CONNECTION_TIMEOUT: + timeout = self.connection_timeout + # print(f'{print_now()}: waiting for {self.name} to connect within {timeout:.4f} s...') + start = time.time() + self._ensure_connected(self._read_pv, self._write_pv, timeout=timeout) + # print(f'{print_now()}: waited for {self.name} to connect for {time.time() - start:.4f} s.') + + EpicsSignalBase.wait_for_connection = wait_for_connection_base + EpicsSignal.wait_for_connection = wait_for_connection + ############################################################################### + + from ophyd.signal import EpicsSignalBase + # EpicsSignalBase.set_default_timeout(timeout=10, connection_timeout=10) # old style + EpicsSignalBase.set_defaults(timeout=10, connection_timeout=10) # new style + +except ImportError: + pass + # Make ophyd listen to pyepics. import logging import nslsii diff --git a/startup/91-callbacks.py b/startup/91-callbacks.py index 2dc391b..73f3c8b 100644 --- a/startup/91-callbacks.py +++ b/startup/91-callbacks.py @@ -33,10 +33,10 @@ def __init__(self, db, data_keys, data_info_keys=None, root='/SHARE/user_data'): self.data_info_keys = data_info_keys self.root = root if data_info_keys is None: - data_info_keys=[ - ('start', 'Proposal ID') - ('start', 'sample_name') - ('start', 'wavelength') + data_info_keys=[ + ('start', 'Proposal ID'), + ('start', 'sample_name'), + ('start', 'wavelength'), ('event', 'data', 'Det_1_Z') ] self.data_info_keys = data_info_keys diff --git a/startup/94-load.py b/startup/94-load.py index 0c1f0c0..cd03c96 100644 --- a/startup/94-load.py +++ b/startup/94-load.py @@ -14,6 +14,8 @@ # ############################################################################## import os +import yaml + from xpdacq.xpdacq_conf import (glbl_dict, configure_device, _reload_glbl, _set_glbl, _load_beamline_config) @@ -54,7 +56,8 @@ xrun.md['beamline_id'] = glbl['beamline_id'] xrun.md['group'] = glbl['group'] xrun.md['facility'] = glbl['facility'] -beamline_config = _load_beamline_config(glbl['blconfig_path']) +with open(glbl['blconfig_path'], "r") as f: + beamline_config = yaml.unsafe_load(f) xrun.md['beamline_config'] = beamline_config # insert header to db, either simulated or real diff --git a/startup/96-dan_functions.py b/startup/96-dan_functions.py index 48a90b7..44a3a60 100644 --- a/startup/96-dan_functions.py +++ b/startup/96-dan_functions.py @@ -11,8 +11,12 @@ ############## -slack_token = os.environ["SLACK_API_TOKEN"] -client = WebClient(token=slack_token) +try: + slack_token = os.environ["SLACK_API_TOKEN"] +except KeyError: + client = None +else: + client = WebClient(token=slack_token) ###