From 1b532961be20112bf8135331c3f878e6947a3120 Mon Sep 17 00:00:00 2001 From: CMS Operator Date: Wed, 14 May 2025 09:52:00 -0400 Subject: [PATCH 01/15] MNT: setup script for DS --- startup/02-data_security.py | 9 +++++++++ 1 file changed, 9 insertions(+) create mode 100644 startup/02-data_security.py diff --git a/startup/02-data_security.py b/startup/02-data_security.py new file mode 100644 index 0000000..6b4896b --- /dev/null +++ b/startup/02-data_security.py @@ -0,0 +1,9 @@ +# To be deleted when DS is done + +RE.md.pop('experiment_alias_directory') +RE.md.pop('experiment_user') +RE.md.pop('experiment_SAF_number') +RE.md.pop('experiment_proposal_number') +RE.md.pop('experiment_project') +RE.md.pop('experiment_group') +RE.md.pop('experiment_cycle') From 3c847313a5dd02914838428caf03c54ee65f37e2 Mon Sep 17 00:00:00 2001 From: CMS Operator Date: Wed, 14 May 2025 09:56:16 -0400 Subject: [PATCH 02/15] FIX: DS script --- startup/02-data_security.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/startup/02-data_security.py b/startup/02-data_security.py index 6b4896b..3b64749 100644 --- a/startup/02-data_security.py +++ b/startup/02-data_security.py @@ -1,9 +1,9 @@ # To be deleted when DS is done -RE.md.pop('experiment_alias_directory') -RE.md.pop('experiment_user') -RE.md.pop('experiment_SAF_number') -RE.md.pop('experiment_proposal_number') -RE.md.pop('experiment_project') -RE.md.pop('experiment_group') -RE.md.pop('experiment_cycle') +RE.md.pop('experiment_alias_directory', None) +RE.md.pop('experiment_user', None) +RE.md.pop('experiment_SAF_number', None) +RE.md.pop('experiment_proposal_number', None) +RE.md.pop('experiment_project', None) +RE.md.pop('experiment_group', None) +RE.md.pop('experiment_cycle', None) From 10ac21d1ce292d0a637f193b1b0123cde370e607 Mon Sep 17 00:00:00 2001 From: CMS Operator Date: Wed, 14 May 2025 15:12:06 -0400 Subject: [PATCH 03/15] WIP: data security 05/14/2025 --- startup/00-startup.py | 247 +++++++++-------------------------- startup/20-area-detectors.py | 153 ++++++++++------------ startup/92-magics.py | 2 +- 3 files changed, 132 insertions(+), 270 deletions(-) diff --git a/startup/00-startup.py b/startup/00-startup.py index 9172316..2723a7d 100644 --- a/startup/00-startup.py +++ b/startup/00-startup.py @@ -1,204 +1,75 @@ print(f'Loading {__file__}') -# import logging -# import caproto -# handler = logging.FileHandler('pilatus-trigger-log.txt') -# from caproto._log import LogFormatter, color_log_format, log_date_format -# handler.setFormatter( -# LogFormatter(color_log_format, datefmt=log_date_format)) -# caproto_log = logging.getLogger('caproto') -# caproto_log.handlers.clear() -# caproto_log.addHandler(handler) -# logging.getLogger('caproto.ch').setLevel('DEBUG') import nslsii -import redis +import os +from tiled.client import from_profile +from databroker import Broker from redis_json_dict import RedisJSONDict -nslsii.configure_base(get_ipython().user_ns, "cms", publish_documents_with_kafka=True, redis_url="info.cms.nsls2.bnl.gov") - -from bluesky.magics import BlueskyMagics -from bluesky.preprocessors import pchain - -# At the end of every run, verify that files were saved and -# print a confirmation message. -from bluesky.callbacks.broker import verify_files_saved - -# RE.subscribe(post_run(verify_files_saved), 'stop') +from IPython import get_ipython +from IPython.terminal.prompts import Prompts, Token + +class ProposalIDPrompt(Prompts): + def in_prompt_tokens(self, cli=None): + return [ + ( + Token.Prompt, + f"{RE.md.get('data_session', 'N/A')} [", + ), + (Token.PromptNum, str(self.shell.execution_count)), + (Token.Prompt, "]: "), + ] + + +ip = get_ipython() +ip.prompts = ProposalIDPrompt(ip) + +# # Configure a Tiled writing client +# tiled_writing_client = from_profile("nsls2", api_key=os.environ["TILED_BLUESKY_WRITING_API_KEY_CMS"])["cms"]["raw"] + +class TiledInserter: + + name = 'cms' + def insert(self, name, doc): + ATTEMPTS = 20 + error = None + for _ in range(ATTEMPTS): + try: + tiled_writing_client.post_document(name, doc) + except Exception as exc: + print("Document saving failure:", repr(exc)) + error = exc + else: + break + time.sleep(2) + else: + # Out of attempts + raise error + +tiled_inserter = TiledInserter() + +nslsii.configure_base(get_ipython().user_ns, + tiled_inserter, + publish_documents_with_kafka=True, + redis_url="info.cms.nsls2.bnl.gov") + +print("Initializing Tiled reading client...\nMake sure you check for duo push.") +tiled_reading_client = from_profile("nsls2", username=None, include_data_sources=True)["cms"]["raw"] + +db = Broker(tiled_reading_client) from pyOlog.ophyd_tools import * -# Uncomment the following lines to turn on verbose messages for -# debugging. -# import logging -# ophyd.logger.setLevel(logging.DEBUG) -# logging.basicConfig(level=logging.DEBUG) - -# Add a callback that prints scan IDs at the start of each scan. -# def print_scan_ids(name, start_doc): -# print("Transient Scan ID: {0} @ {1}".format(start_doc['scan_id'],time.strftime("%Y/%m/%d %H:%M:%S"))) -# print("Persistent Unique Scan ID: '{0}'".format(start_doc['uid'])) -# -# RE.subscribe(print_scan_ids, 'start') - -# - HACK #1 - patch EpicsSignal.get to retry when timeouts happen stolen from HXN -import ophyd - - -def _epicssignal_get(self, *, as_string=None, connection_timeout=1.0, **kwargs): - """Get the readback value through an explicit call to EPICS - Parameters - ---------- - count : int, optional - Explicitly limit count for array data - as_string : bool, optional - Get a string representation of the value, defaults to as_string - from this signal, optional - as_numpy : bool - Use numpy array as the return type for array data. - timeout : float, optional - maximum time to wait for value to be received. - (default = 0.5 + log10(count) seconds) - use_monitor : bool, optional - to use value from latest monitor callback or to make an - explicit CA call for the value. (default: True) - connection_timeout : float, optional - If not already connected, allow up to `connection_timeout` seconds - for the connection to complete. - """ - if as_string is None: - as_string = self._string - - with self._metadata_lock: - if not self._read_pv.connected: - if not self._read_pv.wait_for_connection(connection_timeout): - raise TimeoutError("Failed to connect to %s" % self._read_pv.pvname) - - ret = None - attempts = 0 - max_attempts = 4 - while ret is None and attempts < max_attempts: - attempts += 1 - ret = self._read_pv.get(as_string=as_string, **kwargs) - if ret is None: - print(f"*** PV GET TIMED OUT {self._read_pv.pvname} *** attempt #{attempts}/{max_attempts}") - if ret is None: - print(f"*** PV GET TIMED OUT {self._read_pv.pvname} *** return `None` as value :(") - # TODO we really want to raise TimeoutError here, but that may cause more - # issues in the codebase than we have the time to fix... - # If this causes issues, remove it to keep the old functionality... - raise TimeoutError("Failed to get %s after %d attempts" % (self._read_pv.pvname, attempts)) - if attempts > 1: - print(f"*** PV GET succeeded {self._read_pv.pvname} on attempt #{attempts}") - - if as_string: - return ophyd.signal.waveform_to_string(ret) - - return ret - - -from ophyd import EpicsSignal -from ophyd import EpicsSignalRO - -# from ophyd import EpicsSignalBase - -from ophyd.areadetector import EpicsSignalWithRBV - # Increase the timeout for EpicsSignal.get() # This beamline was occasionally getting ReadTimeoutErrors -# EpicsSignal.set_defaults(timeout=10) -# EpicsSignalRO.set_defaults(timeout=10) +import ophyd ophyd.signal.EpicsSignalBase.set_defaults(timeout=120) - -# We have commented this because we would like to identify the PVs that are causing problems. -# Then the controls group can investigate why it is not working as expected. -# Increasing the get() timeout argument is the prefered way to work around this problem. -# EpicsSignal.get = _epicssignal_get -# EpicsSignalRO.get = _epicssignal_get -# EpicsSignalWithRBV.get = _epicssignal_get - -from pathlib import Path - -import appdirs - - -try: - from bluesky.utils import PersistentDict -except ImportError: - import msgpack - import msgpack_numpy - import zict - - class PersistentDict(zict.Func): - """ - A MutableMapping which syncs it contents to disk. - The contents are stored as msgpack-serialized files, with one file per item - in the mapping. - Note that when an item is *mutated* it is not immediately synced: - >>> d['sample'] = {"color": "red"} # immediately synced - >>> d['sample']['shape'] = 'bar' # not immediately synced - but that the full contents are synced to disk when the PersistentDict - instance is garbage collected. - """ - - def __init__(self, directory): - self._directory = directory - self._file = zict.File(directory) - self._cache = {} - super().__init__(self._dump, self._load, self._file) - self.reload() - - # Similar to flush() or _do_update(), but without reference to self - # to avoid circular reference preventing collection. - # NOTE: This still doesn't guarantee call on delete or gc.collect()! - # Explicitly call flush() if immediate write to disk required. - def finalize(zfile, cache, dump): - zfile.update((k, dump(v)) for k, v in cache.items()) - - import weakref - - self._finalizer = weakref.finalize(self, finalize, self._file, self._cache, PersistentDict._dump) - - @property - def directory(self): - return self._directory - - def __setitem__(self, key, value): - self._cache[key] = value - super().__setitem__(key, value) - - def __getitem__(self, key): - return self._cache[key] - - def __delitem__(self, key): - del self._cache[key] - super().__delitem__(key) - - def __repr__(self): - return f"<{self.__class__.__name__} {dict(self)!r}>" - - @staticmethod - def _dump(obj): - "Encode as msgpack using numpy-aware encoder." - # See https://github.com/msgpack/msgpack-python#string-and-binary-type - # for more on use_bin_type. - return msgpack.packb(obj, default=msgpack_numpy.encode, use_bin_type=True) - - @staticmethod - def _load(file): - return msgpack.unpackb(file, object_hook=msgpack_numpy.decode, raw=False) - - def flush(self): - """Force a write of the current state to disk""" - for k, v in self.items(): - super().__setitem__(k, v) - - def reload(self): - """Force a reload from disk, overwriting current cache""" - self._cache = dict(super().items()) - - #this replaces RE() < from bluesky.utils import register_transform register_transform('RE', prefix='<') + +# Setup the path to the secure assets folder for the current proposal +def assets_path(): + return f"/nsls2/data/cms/proposals/{RE.md['cycle']}/{RE.md['data_session']}/assets/" \ No newline at end of file diff --git a/startup/20-area-detectors.py b/startup/20-area-detectors.py index b41ffba..cb20ac0 100644 --- a/startup/20-area-detectors.py +++ b/startup/20-area-detectors.py @@ -225,8 +225,8 @@ def setExposureTime(self, exposure_time, verbosity=3): yield from mv( self.cam.acquire_time, exposure_time, - self.cam.acquire_period, - exposure_time + 0.1, + # self.cam.acquire_period, + # exposure_time + 0.1, ) # caput('XF:11BMB-ES{Det:SAXS}:cam1:AcquireTime', exposure_time) # caput('XF:11BMB-ES{Det:SAXS}:cam1:AcquirePeriod', exposure_time+0.1) @@ -238,17 +238,6 @@ def setExposureNumber(self, exposure_number, verbosity=3): yield from mv(self.cam.num_images, exposure_number) -# class Pilatus800V33(PilatusV33): -# tiff = Cpt(TIFFPluginWithFileStore, -# suffix='TIFF1:', -# # write_path_template='/Pilatus800K/%Y/%m/%d/', -# # read_path_template='/nsls2/xf11bm/Pilatus800K/%Y/%m/%d/', -# # root='/nsls2/xf11bm') -# write_path_template='/nsls2/data/cms/legacy/xf11bm/Pilatus800/%Y/%m/%d/', -# # read_path_template='/nsls2/data/cms/legacy/xf11bm/Pilatus800/%Y/%m/%d/', -# root='/nsls2/data/cms/legacy/xf11bm') - - class Pilatus800V33(SingleTriggerV33, PilatusDetector): cam = Cpt(PilatusDetectorCamV33, "cam1:") image = Cpt(ImagePlugin, "image1:") @@ -266,21 +255,20 @@ class Pilatus800V33(SingleTriggerV33, PilatusDetector): tiff = Cpt( TIFFPluginWithFileStore, suffix="TIFF1:", - # write_path_template='/nsls2/xf11bm/Pilatus2M/%Y/%m/%d/', # GPFS client - # write_path_template='/Pilatus2M/%Y/%m/%d/', # NSF-mount of GPFS directory - # root='/nsls2/xf11bm' - read_path_template="/nsls2/data/cms/legacy/xf11bm/Pilatus800/%Y/%m/%d/", - write_path_template="/nsls2/data/cms/legacy/xf11bm/Pilatus800/%Y/%m/%d/", - root="/nsls2/data/cms/legacy/xf11bm", ) - # root='/') + + def stage(self, *args, **kwargs): + self.tiff.write_path_template = assets_path() + f'{self.name}/%Y/%m/%d/' + self.tiff.read_path_template = assets_path() + f'{self.name}/%Y/%m/%d/' + self.tiff.reg_root = assets_path() + f'{self.name}' + return super().stage(*args, **kwargs) def setExposureTime(self, exposure_time, verbosity=3): yield from mv( self.cam.acquire_time, exposure_time, - self.cam.acquire_period, - exposure_time + 0.1, + # self.cam.acquire_period, + # exposure_time + 0.1, ) # self.cam.acquire_time.put(exposure_time) # self.cam.acquire_period.put(exposure_time+.1) @@ -299,15 +287,17 @@ class Pilatus8002V33(PilatusV33): tiff = Cpt( TIFFPluginWithFileStore, suffix="TIFF1:", - # write_path_template='/Pilatus800K/%Y/%m/%d/', - # read_path_template='/nsls2/xf11bm/Pilatus800K/%Y/%m/%d/', - # root='/nsls2/xf11bm') - write_path_template="/nsls2/data/cms/legacy/xf11bm/Pilatus800_2/%Y/%m/%d/", - root="/nsls2/data/cms/legacy/xf11bm", # write_path_template="/ramdisk/", # root="/ramdisk/", ) + def stage(self, *args, **kwargs): + self.tiff.write_path_template = assets_path() + f'{self.name}/%Y/%m/%d/' + self.tiff.read_path_template = assets_path() + f'{self.name}/%Y/%m/%d/' + self.tiff.reg_root = assets_path() + f'{self.name}' + return super().stage(*args, **kwargs) + + # class Pilatus300V33(PilatusV33): # tiff = Cpt( @@ -340,10 +330,14 @@ class Pilatus2M(SingleTrigger, PilatusDetector): tiff = Cpt( TIFFPluginWithFileStore, suffix="TIFF1:", - write_path_template="/nsls2/xf11bm/Pilatus2M/%Y/%m/%d/", - root="/nsls2/xf11bm", ) + def stage(self, *args, **kwargs): + self.tiff.write_path_template = assets_path() + f'{self.name}/%Y/%m/%d/' + self.tiff.read_path_template = assets_path() + f'{self.name}/%Y/%m/%d/' + self.tiff.reg_root = assets_path() + f'{self.name}' + return super().stage(*args, **kwargs) + def setExposureTime(self, exposure_time, verbosity=3): # how to do this with stage_sigs (warning, need to change this every time # if you set) @@ -378,13 +372,14 @@ class Pilatus2MV33(SingleTriggerV33, PilatusDetector): tiff = Cpt( TIFFPluginWithFileStore, suffix="TIFF1:", - # write_path_template='/nsls2/xf11bm/Pilatus2M/%Y/%m/%d/', # GPFS client - # write_path_template='/Pilatus2M/%Y/%m/%d/', # NSF-mount of GPFS directory - # root='/nsls2/xf11bm' - write_path_template="/nsls2/data/cms/legacy/xf11bm/Pilatus2M/%Y/%m/%d/", # Lustre client - root="/nsls2/data/cms/legacy/xf11bm", ) - # root='/') + + def stage(self, *args, **kwargs): + self.tiff.write_path_template = assets_path() + f'{self.name}/%Y/%m/%d/' + self.tiff.read_path_template = assets_path() + f'{self.name}/%Y/%m/%d/' + self.tiff.reg_root = assets_path() + f'{self.name}' + return super().stage(*args, **kwargs) + def setExposureTime(self, exposure_time, verbosity=3): yield from mv( self.cam.acquire_time, @@ -427,20 +422,14 @@ class Pilatus2MV33_h5(SingleTriggerV33, PilatusDetector): h5 = Cpt( HDF5PluginWithFileStore, suffix="HDF1:", - # write_path_template='/nsls2/xf11bm/Pilatus2M/%Y/%m/%d/', # GPFS client - # write_path_template='/Pilatus2M/%Y/%m/%d/', # NSF-mount of GPFS directory - # root='/nsls2/xf11bm' - write_path_template="/nsls2/data/cms/legacy/xf11bm/Pilatus2M/%Y/%m/%d/", # Lustre client - root="/nsls2/data/cms/legacy/xf11bm", ) - def setExposureTime(self, exposure_time, verbosity=3): yield from mv( self.cam.acquire_time, exposure_time, - self.cam.acquire_period, - exposure_time + 0.1, + # self.cam.acquire_period, + # exposure_time + 0.1, ) # self.cam.acquire_time.put(exposure_time) # self.cam.acquire_period.put(exposure_time+.1) @@ -454,8 +443,12 @@ def setExposureNumber(self, exposure_number, verbosity=3): yield from mv(self.cam.num_images, exposure_number) def stage(self): - error = None + self.h5.write_path_template = assets_path() + f'{self.name}/%Y/%m/%d/' + self.h5.read_path_template = assets_path() + f'{self.name}/%Y/%m/%d/' + self.h5.reg_root = assets_path() + f'{self.name}' + # wrap the staging process in a retry loop + error = None for retry in range(5): try: return super().stage() @@ -465,13 +458,13 @@ def stage(self): # Stash the exception as the variable 'error' error = err else: - # Staging worked. Strop retyring. + # Staging worked. Stop retyring. break else: # We exhausted all retires and none worked. # Raise the error captured above to produce a useful error message. raise error - + return super().stage(*args, **kwargs) # print( 'This is the 20-area-dec py.') # class StandardProsilicaWithTIFF(StandardProsilica): @@ -530,42 +523,42 @@ def stage(self): - fs1.tiff.read_attrs = [] - fs1.stats3.total.kind = "hinted" - fs1.stats4.total.kind = "hinted" - STATS_NAMES = ["stats1", "stats2", "stats3", "stats4", "stats5"] - fs1.read_attrs = ["tiff"] + STATS_NAMES - for stats_name in STATS_NAMES: - stats_plugin = getattr(fs1, stats_name) - stats_plugin.read_attrs = ["total"] + # fs1.tiff.read_attrs = [] + # fs1.stats3.total.kind = "hinted" + # fs1.stats4.total.kind = "hinted" + # STATS_NAMES = ["stats1", "stats2", "stats3", "stats4", "stats5"] + # fs1.read_attrs = ["tiff"] + STATS_NAMES + # for stats_name in STATS_NAMES: + # stats_plugin = getattr(fs1, stats_name) + # stats_plugin.read_attrs = ["total"] - for item in fs1.stats1.configuration_attrs: - item_check = getattr(fs1.stats1, item) - item_check.kind = "omitted" + # for item in fs1.stats1.configuration_attrs: + # item_check = getattr(fs1.stats1, item) + # item_check.kind = "omitted" - for item in fs1.stats2.configuration_attrs: - item_check = getattr(fs1.stats2, item) - item_check.kind = "omitted" + # for item in fs1.stats2.configuration_attrs: + # item_check = getattr(fs1.stats2, item) + # item_check.kind = "omitted" - for item in fs1.stats3.configuration_attrs: - item_check = getattr(fs1.stats3, item) - item_check.kind = "omitted" + # for item in fs1.stats3.configuration_attrs: + # item_check = getattr(fs1.stats3, item) + # item_check.kind = "omitted" - for item in fs1.stats4.configuration_attrs: - item_check = getattr(fs1.stats4, item) - item_check.kind = "omitted" + # for item in fs1.stats4.configuration_attrs: + # item_check = getattr(fs1.stats4, item) + # item_check.kind = "omitted" - for item in fs1.stats5.configuration_attrs: - item_check = getattr(fs1.stats5, item) - item_check.kind = "omitted" + # for item in fs1.stats5.configuration_attrs: + # item_check = getattr(fs1.stats5, item) + # item_check.kind = "omitted" - for item in fs1.tiff.configuration_attrs: - item_check = getattr(fs1.tiff, item) - item_check.kind = "omitted" + # for item in fs1.tiff.configuration_attrs: + # item_check = getattr(fs1.tiff, item) + # item_check.kind = "omitted" - for item in fs1.cam.configuration_attrs: - item_check = getattr(fs1.cam, item) - item_check.kind = "omitted" + # for item in fs1.cam.configuration_attrs: + # item_check = getattr(fs1.cam, item) + # item_check.kind = "omitted" @@ -641,10 +634,8 @@ def stage(self): pilatus300 = "Pil300ISNOTWORKING" # pilatus800 section -# if False: -# if True: if Pilatus800_on == True: - pilatus800 = Pilatus800V33("XF:11BMB-ES{Det:PIL800K}:", name="pilatus800") + pilatus800 = Pilatus800V33("XF:11BMB-ES{Det:PIL800K}:", name="pilatus800k-1") pilatus800.tiff.read_attrs = [] pilatus800.stats3.total.kind = "hinted" pilatus800.stats4.total.kind = "hinted" @@ -689,7 +680,7 @@ def stage(self): # if True: if Pilatus800_2_on == True: # TODO: - pilatus8002 = Pilatus8002V33("XF:11BMB-ES{Det:PIL800K2}:", name="pilatus8002") # change PV + pilatus8002 = Pilatus8002V33("XF:11BMB-ES{Det:PIL800K2}:", name="pilatus800k-2") # change PV pilatus8002.tiff.read_attrs = [] pilatus8002.stats3.total.kind = "hinted" pilatus8002.stats4.total.kind = "hinted" @@ -732,7 +723,7 @@ def stage(self): # pilatus2M section # if False: if Pilatus2M_on == True: - pilatus2M = Pilatus2MV33("XF:11BMB-ES{Det:PIL2M}:", name="pilatus2M") + pilatus2M = Pilatus2MV33("XF:11BMB-ES{Det:PIL2M}:", name="pilatus2m-1") pilatus2M.tiff.read_attrs = [] STATS_NAMES2M = ["stats1", "stats2", "stats3", "stats4"] @@ -775,7 +766,7 @@ def stage(self): elif Pilatus2M_on == 'h5': - pilatus2M = Pilatus2MV33_h5("XF:11BMB-ES{Det:PIL2M}:", name="pilatus2M") + pilatus2M = Pilatus2MV33_h5("XF:11BMB-ES{Det:PIL2M}:", name="pilatus2m-1") pilatus2M.h5.read_attrs = [] STATS_NAMES2M = ["stats1", "stats2", "stats3", "stats4"] diff --git a/startup/92-magics.py b/startup/92-magics.py index 7b2230e..dfb0487 100644 --- a/startup/92-magics.py +++ b/startup/92-magics.py @@ -1,4 +1,4 @@ -# BlueskyMagics were imported and registered in 00-startup.py +from bluesky.magics import BlueskyMagics BlueskyMagics.detectors = [pilatus2M] BlueskyMagics.positioners = [ From 6fd442e444c17c4fd266a1654dd1f768be016646 Mon Sep 17 00:00:00 2001 From: CMS Operator Date: Thu, 15 May 2025 09:47:20 -0400 Subject: [PATCH 04/15] WIP: tests of Pilatus detectors --- acceptance_tests/01_count_pilatus300.py | 14 -------------- startup/00-startup.py | 8 ++++---- startup/20-area-detectors.py | 4 ++++ 3 files changed, 8 insertions(+), 18 deletions(-) delete mode 100644 acceptance_tests/01_count_pilatus300.py diff --git a/acceptance_tests/01_count_pilatus300.py b/acceptance_tests/01_count_pilatus300.py deleted file mode 100644 index f737a85..0000000 --- a/acceptance_tests/01_count_pilatus300.py +++ /dev/null @@ -1,14 +0,0 @@ -from bluesky.plans import count -from bluesky.plans import bps - - -def pil_test(): - uid = RE(count([pilatus2M])) - h = db[uid[0]] - # print(uid) - print(h.start) - return h - - -def motor_test(pos=0): - RE(bps.mov(smx, pos)) diff --git a/startup/00-startup.py b/startup/00-startup.py index 2723a7d..21e5fca 100644 --- a/startup/00-startup.py +++ b/startup/00-startup.py @@ -21,7 +21,6 @@ def in_prompt_tokens(self, cli=None): (Token.Prompt, "]: "), ] - ip = get_ipython() ip.prompts = ProposalIDPrompt(ip) @@ -48,16 +47,17 @@ def insert(self, name, doc): raise error tiled_inserter = TiledInserter() +tiled_inserter = "cms" nslsii.configure_base(get_ipython().user_ns, tiled_inserter, publish_documents_with_kafka=True, redis_url="info.cms.nsls2.bnl.gov") -print("Initializing Tiled reading client...\nMake sure you check for duo push.") -tiled_reading_client = from_profile("nsls2", username=None, include_data_sources=True)["cms"]["raw"] +# print("Initializing Tiled reading client...\nMake sure you check for duo push.") +# tiled_reading_client = from_profile("nsls2", username=None, include_data_sources=True)["cms"]["raw"] -db = Broker(tiled_reading_client) +# db = Broker(tiled_reading_client) from pyOlog.ophyd_tools import * diff --git a/startup/20-area-detectors.py b/startup/20-area-detectors.py index cb20ac0..cb8994c 100644 --- a/startup/20-area-detectors.py +++ b/startup/20-area-detectors.py @@ -255,6 +255,7 @@ class Pilatus800V33(SingleTriggerV33, PilatusDetector): tiff = Cpt( TIFFPluginWithFileStore, suffix="TIFF1:", + write_path_template = "", ) def stage(self, *args, **kwargs): @@ -330,6 +331,7 @@ class Pilatus2M(SingleTrigger, PilatusDetector): tiff = Cpt( TIFFPluginWithFileStore, suffix="TIFF1:", + write_path_template = "", ) def stage(self, *args, **kwargs): @@ -372,6 +374,7 @@ class Pilatus2MV33(SingleTriggerV33, PilatusDetector): tiff = Cpt( TIFFPluginWithFileStore, suffix="TIFF1:", + write_path_template = "", ) def stage(self, *args, **kwargs): @@ -422,6 +425,7 @@ class Pilatus2MV33_h5(SingleTriggerV33, PilatusDetector): h5 = Cpt( HDF5PluginWithFileStore, suffix="HDF1:", + write_path_template = "", ) def setExposureTime(self, exposure_time, verbosity=3): From 1e86c51430ca98b6312c07866af2f04d8b18d02c Mon Sep 17 00:00:00 2001 From: CMS Operator Date: Thu, 15 May 2025 17:38:53 -0400 Subject: [PATCH 05/15] WIP: configured detector paths --- acceptance_tests/01_count_pilatus.py | 14 ++++ startup/00-startup.py | 6 +- startup/20-area-detectors.py | 121 +++++++-------------------- 3 files changed, 46 insertions(+), 95 deletions(-) create mode 100644 acceptance_tests/01_count_pilatus.py diff --git a/acceptance_tests/01_count_pilatus.py b/acceptance_tests/01_count_pilatus.py new file mode 100644 index 0000000..f737a85 --- /dev/null +++ b/acceptance_tests/01_count_pilatus.py @@ -0,0 +1,14 @@ +from bluesky.plans import count +from bluesky.plans import bps + + +def pil_test(): + uid = RE(count([pilatus2M])) + h = db[uid[0]] + # print(uid) + print(h.start) + return h + + +def motor_test(pos=0): + RE(bps.mov(smx, pos)) diff --git a/startup/00-startup.py b/startup/00-startup.py index 21e5fca..fb24b23 100644 --- a/startup/00-startup.py +++ b/startup/00-startup.py @@ -54,10 +54,10 @@ def insert(self, name, doc): publish_documents_with_kafka=True, redis_url="info.cms.nsls2.bnl.gov") -# print("Initializing Tiled reading client...\nMake sure you check for duo push.") -# tiled_reading_client = from_profile("nsls2", username=None, include_data_sources=True)["cms"]["raw"] +print("Initializing Tiled reading client...\nMake sure you check for duo push.") +tiled_reading_client = from_profile("nsls2", username=None, include_data_sources=True)["cms"]["raw"] -# db = Broker(tiled_reading_client) +db = Broker(tiled_reading_client) from pyOlog.ophyd_tools import * diff --git a/startup/20-area-detectors.py b/startup/20-area-detectors.py index cb8994c..f221fb5 100644 --- a/startup/20-area-detectors.py +++ b/startup/20-area-detectors.py @@ -54,7 +54,7 @@ Pilatus800_2_on = False # Pilatus800_on = False -# Pilatus800_2_on = True +Pilatus800_2_on = True # Pilatus800_2_on = False # Pilatus2M_on = False @@ -87,34 +87,10 @@ def ensure_nonblocking(self): cpt.ensure_nonblocking() -# class Pilatus2M(SingleTrigger, PilatusDetector): -# image = Cpt(ImagePlugin, 'image1:') -# stats1 = Cpt(StatsPlugin, 'Stats1:') -# stats2 = Cpt(StatsPlugin, 'Stats2:') -# stats3 = Cpt(StatsPlugin, 'Stats3:') -# stats4 = Cpt(StatsPlugin, 'Stats4:') -# stats5 = Cpt(StatsPlugin, 'Stats5:') -# roi1 = Cpt(ROIPlugin, 'ROI1:') -# roi2 = Cpt(ROIPlugin, 'ROI2:') -# roi3 = Cpt(ROIPlugin, 'ROI3:') -# roi4 = Cpt(ROIPlugin, 'ROI4:') -# proc1 = Cpt(ProcessPlugin, 'Proc1:') - -# tiff = Cpt(TIFFPluginWithFileStore, -# suffix='TIFF1:', -# # write_path_template='/nsls2/data/cms/legacy/xf11bm/Pilatus2M/%Y/%m/%d/', -# write_path_template='/nsls2/xf11bm/Pilatus2M/%Y/%m/%d/', -# root='/nsls2/xf11bm') - -# def setExposureTime(self, exposure_time, verbosity=3): -# caput('XF:11BMB-ES{Det:PIL2M}:cam1:AcquireTime', exposure_time) -# caput('XF:11BMB-ES{Det:PIL2M}:cam1:AcquirePeriod', exposure_time+0.1) - - class StandardProsilica(SingleTrigger, ProsilicaDetector): - # tiff = Cpt(TIFFPluginWithFileStore, - # suffix='TIFF1:', - # write_path_template='/XF11ID/data/') + tiff = Cpt(TIFFPluginWithFileStore, + suffix='TIFF1:', + write_path_template='',) image = Cpt(ImagePlugin, "image1:") stats1 = Cpt(StatsPluginV33, "Stats1:") stats2 = Cpt(StatsPluginV33, "Stats2:") @@ -128,11 +104,17 @@ class StandardProsilica(SingleTrigger, ProsilicaDetector): roi4 = Cpt(ROIPlugin, "ROI4:") proc1 = Cpt(ProcessPlugin, "Proc1:") + def stage(self, *args, **kwargs): + self.tiff.write_path_template = assets_path() + f'{self.name}/%Y/%m/%d/' + self.tiff.read_path_template = assets_path() + f'{self.name}/%Y/%m/%d/' + self.tiff.reg_root = assets_path() + f'{self.name}' + return super().stage(*args, **kwargs) + class StandardProsilicaV33(SingleTriggerV33, ProsilicaDetector): - # tiff = Cpt(TIFFPluginWithFileStore, - # suffix='TIFF1:', - # write_path_template='/XF11ID/data/') + tiff = Cpt(TIFFPluginWithFileStore, + suffix='TIFF1:', + write_path_template='',) cam = Cpt(ProsilicaDetectorCamV33, "cam1:") image = Cpt(ImagePlugin, "image1:") stats1 = Cpt(StatsPluginV33, "Stats1:") @@ -147,16 +129,11 @@ class StandardProsilicaV33(SingleTriggerV33, ProsilicaDetector): roi4 = Cpt(ROIPlugin, "ROI4:") proc1 = Cpt(ProcessPlugin, "Proc1:") - # tiff = Cpt( - # TIFFPluginWithFileStore, - # suffix="TIFF1:", - # # write_path_template='/nsls2/xf11bm/Pilatus2M/%Y/%m/%d/', # GPFS client - # # write_path_template='/Pilatus2M/%Y/%m/%d/', # NSF-mount of GPFS directory - # # root='/nsls2/xf11bm' - # read_path_template="/nsls2/data/cms/legacy/xf11bm/Pilatus2M/%Y/%m/%d/", - # write_path_template="/nsls2/data/cms/legacy/xf11bm/Pilatus2M/%Y/%m/%d/", - # root="/nsls2/data/cms/legacy/xf11bm", - # ) + def stage(self, *args, **kwargs): + self.tiff.write_path_template = assets_path() + f'{self.name}/%Y/%m/%d/' + self.tiff.read_path_template = assets_path() + f'{self.name}/%Y/%m/%d/' + self.tiff.reg_root = assets_path() + f'{self.name}' + return super().stage(*args, **kwargs) class PilatusDetectorCamV33(PilatusDetectorCam): """This is used to update the standard prosilica to AD33.""" @@ -177,29 +154,6 @@ def ensure_nonblocking(self): cpt.ensure_nonblocking() -# class Pilatus(SingleTrigger, PilatusDetector): -# image = Cpt(ImagePlugin, 'image1:') -# stats1 = Cpt(StatsPluginV33, 'Stats1:') -# stats2 = Cpt(StatsPluginV33, 'Stats2:') -# stats3 = Cpt(StatsPluginV33, 'Stats3:') -# stats4 = Cpt(StatsPluginV33, 'Stats4:') -# stats5 = Cpt(StatsPluginV33, 'Stats5:') -# roi1 = Cpt(ROIPlugin, 'ROI1:') -# roi2 = Cpt(ROIPlugin, 'ROI2:') -# roi3 = Cpt(ROIPlugin, 'ROI3:') -# roi4 = Cpt(ROIPlugin, 'ROI4:') -# proc1 = Cpt(ProcessPlugin, 'Proc1:') - -# tiff = Cpt(TIFFPluginWithFileStore, -# suffix='TIFF1:', -# write_path_template='/nsls2/xf11bm/Pilatus300/%Y/%m/%d/', -# root='/nsls2/xf11bm') - -# def setExposureTime(self, exposure_time, verbosity=3): -# caput('XF:11BMB-ES{Det:SAXS}:cam1:AcquireTime', exposure_time) -# caput('XF:11BMB-ES{Det:SAXS}:cam1:AcquirePeriod', exposure_time+0.1) - - class PilatusV33(SingleTriggerV33, PilatusDetector): cam = Cpt(PilatusDetectorCamV33, "cam1:") image = Cpt(ImagePlugin, "image1:") @@ -288,7 +242,7 @@ class Pilatus8002V33(PilatusV33): tiff = Cpt( TIFFPluginWithFileStore, suffix="TIFF1:", - # write_path_template="/ramdisk/", + write_path_template="", # "/ramdisk/", # root="/ramdisk/", ) @@ -470,28 +424,6 @@ def stage(self): raise error return super().stage(*args, **kwargs) -# print( 'This is the 20-area-dec py.') -# class StandardProsilicaWithTIFF(StandardProsilica): -# tiff = Cpt(TIFFPluginWithFileStore, -# suffix='TIFF1:', -# write_path_template='/nsls2/xf11bm/data/%Y/%m/%d/', -# root='/nsls2/xf11bm/') - - -## This renaming should be reversed: no correspondance between CSS screens, PV names and ophyd.... -# xray_eye1 = StandardProsilica('XF:11IDA-BI{Bpm:1-Cam:1}', name='xray_eye1') -# xray_eye2 = StandardProsilica('XF:11IDB-BI{Mon:1-Cam:1}', name='xray_eye2') -# xray_eye3 = StandardProsilica('XF:11IDB-BI{Cam:08}', name='xray_eye3') -# xray_eye1_writing = StandardProsilicaWithTIFF('XF:11IDA-BI{Bpm:1-Cam:1}', name='xray_eye1') -# xray_eye2_writing = StandardProsilicaWithTIFF('XF:11IDB-BI{Mon:1-Cam:1}', name='xray_eye2') -# xray_eye3_writing = StandardProsilicaWithTIFF('XF:11IDB-BI{Cam:08}', name='xray_eye3') -# fs1 = StandardProsilica('XF:11IDA-BI{FS:1-Cam:1}', name='fs1') -# fs2 = StandardProsilica('XF:11IDA-BI{FS:2-Cam:1}', name='fs2') -# fs_wbs = StandardProsilica('XF:11IDA-BI{BS:WB-Cam:1}', name='fs_wbs') -# dcm_cam = StandardProsilica('XF:11IDA-BI{Mono:DCM-Cam:1}', name='dcm_cam') -# fs_pbs = StandardProsilica('XF:11IDA-BI{BS:PB-Cam:1}', name='fs_pbs') -# elm = Elm('XF:11IDA-BI{AH401B}AH401B:',) - import time @@ -499,14 +431,19 @@ def stage(self): if Camera_on==True: # time.sleep(1) - fs1 = StandardProsilicaV33('XF:11BMA-BI{FS:1-Cam:1}', name='fs1') + fs1 = StandardProsilicaV33('XF:11BMA-BI{FS:1-Cam:1}', name='webcam-1') # time.sleep(1) - fs2 = StandardProsilicaV33("XF:11BMA-BI{FS:2-Cam:1}", name="fs2") + fs2 = StandardProsilicaV33("XF:11BMA-BI{FS:2-Cam:1}", name="webcam-2") # time.sleep(1) - fs3 = StandardProsilicaV33("XF:11BMB-BI{FS:3-Cam:1}", name="fs3") + fs3 = StandardProsilicaV33("XF:11BMB-BI{FS:3-Cam:1}", name="webcam-3") # time.sleep(1) - fs4 = StandardProsilicaV33("XF:11BMB-BI{FS:4-Cam:1}", name="fs4") - fs5 = StandardProsilicaV33("XF:11BMB-BI{FS:Test-Cam:1}", name="fs5") + fs4 = StandardProsilicaV33("XF:11BMB-BI{FS:4-Cam:1}", name="webcam-4") + # mobile camera + fs5 = StandardProsilicaV33("XF:11BMB-BI{FS:Test-Cam:1}", name="webcam-5") + # off-axis camera + fs6 = StandardProsilicaV33("XF:11BMB-BI{OffAxis-Cam:1}", name="webcam-6") + # on-axis camera + fs9 = StandardProsilicaV33("XF:11BMB-BI{OnAxis-Cam:2}", name="webcam-9") all_standard_pros = [fs2, fs3, fs4] From 08aa79defd714a21c4158b3ecc4127bee075e998 Mon Sep 17 00:00:00 2001 From: CMS Operator Date: Fri, 16 May 2025 11:22:12 -0400 Subject: [PATCH 06/15] WIP: cameras writing to Tiled --- .../94-sample-checkpoint.py | 44 ------ startup/00-startup.py | 7 +- startup/20-area-detectors.py | 129 ++++++------------ startup/86-live-spec.py | 2 +- startup/90-bluesky.py | 3 - startup/94-sample.py | 23 ---- startup/user_collection/user_HighVoltage.py | 2 +- startup/user_collection/user_LinkamTensile.py | 2 +- startup/user_collection/user_LinkamThermal.py | 2 +- .../user_TSAXSWAXS_stitching.py | 2 +- .../user_openMAXS.py.LZhu2022C1 | 2 +- 11 files changed, 53 insertions(+), 165 deletions(-) diff --git a/startup/.ipynb_checkpoints/94-sample-checkpoint.py b/startup/.ipynb_checkpoints/94-sample-checkpoint.py index 743c62c..8206e4c 100644 --- a/startup/.ipynb_checkpoints/94-sample-checkpoint.py +++ b/startup/.ipynb_checkpoints/94-sample-checkpoint.py @@ -1940,53 +1940,9 @@ def expose(self, exposure_time=None, extra=None, handlefile=True, verbosity=3, p percentage = 100 * (time.time() - start_time) / max_exposure_time print("After re-exposing .... percentage = {} ".format(percentage)) - # if detector.name is 'pilatus300': - # if caget('XF:11BMB-ES{Det:SAXS}:cam1:Acquire')==1: - # status *= 0 - # elif detector.name is 'pilatus2M': - # if caget('XF:11BMB-ES{Det:PIL2M}:cam1:Acquire')==1: - # status *= 0 - # elif detector.name is 'pilatus800': - # if caget('XF:11BMB-ES{Det:PIL800K}:cam1:Acquire')==1: - # status *= 0 - # elif detector.name is 'PhotonicSciences_CMS': - # if not detector.detector_is_ready(verbosity=0): - # status *= 0 - - # if verbosity>=3 and caget('XF:11BMB-ES{Det:PIL800K}:cam1:Acquire')==1: - # print('Warning: Detector pilatus300 still not done acquiring.') - - # #if verbosity>=3 and caget('XF:11BMB-ES{Det:SAXS}:cam1:Acquire')==1: - # #print('Warning: Detector pilatus300 still not done acquiring.') - - # if verbosity>=3 and caget('XF:11BMB-ES{Det:PIL2M}:cam1:Acquire')==1: - # print('Warning: Detector pilatus2M still not done acquiring.') get_beamline().beam.off() - # save the percentage information - # if verbosity>=5: - # folder = '/nsls2/data/cms/legacy/xf11bm/data/2022_1/TKoga2/' - # # filename = '' - - # current_data = {'a_sample': self.name, - # 'b_exposure_time': detector.cam.acquire_time.get(), - # 'c_exposure_percentage': percentage, - # 'd_align_time': md['filename'] - # } - - # temp_data = pds.DataFrame([current_data]) - - # # INT_FILENAME='{}/data/{}.csv'.format(os.path.dirname(__file__) , 'alignment_results.csv') - # INT_FILENAME='{}/data/{}.csv'.format(folder , 'exposure_info.csv') - - # if os.path.isfile(INT_FILENAME): - # output_data = pds.read_csv(INT_FILENAME, index_col=0) - # output_data = output_data.append(temp_data, ignore_index=True) - # output_data.to_csv(INT_FILENAME) - # else: - # temp_data.to_csv(INT_FILENAME) - if handlefile == True: for detector in get_beamline().detector: self.handle_file(detector, extra=extra, verbosity=verbosity, **md) diff --git a/startup/00-startup.py b/startup/00-startup.py index fb24b23..4427948 100644 --- a/startup/00-startup.py +++ b/startup/00-startup.py @@ -57,7 +57,7 @@ def insert(self, name, doc): print("Initializing Tiled reading client...\nMake sure you check for duo push.") tiled_reading_client = from_profile("nsls2", username=None, include_data_sources=True)["cms"]["raw"] -db = Broker(tiled_reading_client) +# db = Broker(tiled_reading_client) from pyOlog.ophyd_tools import * @@ -71,5 +71,8 @@ def insert(self, name, doc): register_transform('RE', prefix='<') # Setup the path to the secure assets folder for the current proposal +def proposal_path(): + return f"/nsls2/data/cms/proposals/{RE.md['cycle']}/{RE.md['data_session']}/" + def assets_path(): - return f"/nsls2/data/cms/proposals/{RE.md['cycle']}/{RE.md['data_session']}/assets/" \ No newline at end of file + return proposal_path() + "assets/" \ No newline at end of file diff --git a/startup/20-area-detectors.py b/startup/20-area-detectors.py index f221fb5..7f73ef9 100644 --- a/startup/20-area-detectors.py +++ b/startup/20-area-detectors.py @@ -1,5 +1,5 @@ -# import time as ttime # tea time -# from datetime import datetime +print(f'Loading {__file__}') + from ophyd import ( ProsilicaDetector, SingleTrigger, @@ -28,14 +28,7 @@ from ophyd.areadetector.filestore_mixins import FileStoreHDF5IterativeWrite from ophyd.areadetector.plugins import HDF5Plugin #,register_plugin,PluginBase - -print(f'Loading {__file__}') - -# import filestore.api as fs - - -# class Elm(SingleTrigger, DetectorBase): -# pass +import time Pilatus2M_on = True @@ -61,9 +54,30 @@ ''' H5Plugin = HDF5Plugin ''' - class TIFFPluginWithFileStore(TIFFPlugin, FileStoreTIFFIterativeWrite): - pass + def describe(self): + ret = super().describe() + key = self.parent._image_name + color_mode = self.parent.cam.color_mode.get(as_string=True) + if color_mode == 'Mono': + ret[key]['shape'] = [ + self.parent.cam.num_images.get(), + #self.array_size.depth.get(), + self.array_size.height.get(), + self.array_size.width.get() + ] + + elif color_mode in ['RGB1', 'Bayer']: + ret[key]['shape'] = [self.parent.cam.num_images.get(), *self.array_size.get()] + else: + raise RuntimeError("Color mode not supported") + + cam_dtype = self.parent.cam.data_type.get(as_string=True) + type_map = {'UInt8': '|u1', 'UInt16': '=5: - # folder = '/nsls2/data/cms/legacy/xf11bm/data/2022_1/TKoga2/' - # # filename = '' - - # current_data = {'a_sample': self.name, - # 'b_exposure_time': detector.cam.acquire_time.get(), - # 'c_exposure_percentage': percentage, - # 'd_align_time': md['filename'] - # } - - # temp_data = pds.DataFrame([current_data]) - - # # INT_FILENAME='{}/data/{}.csv'.format(os.path.dirname(__file__) , 'alignment_results.csv') - # INT_FILENAME='{}/data/{}.csv'.format(folder , 'exposure_info.csv') - - # if os.path.isfile(INT_FILENAME): - # output_data = pds.read_csv(INT_FILENAME, index_col=0) - # output_data = output_data.append(temp_data, ignore_index=True) - # output_data.to_csv(INT_FILENAME) - # else: - # temp_data.to_csv(INT_FILENAME) - if handlefile == True: for detector in get_beamline().detector: self.handle_file(detector, extra=extra, verbosity=verbosity, **md) diff --git a/startup/user_collection/user_HighVoltage.py b/startup/user_collection/user_HighVoltage.py index 4d24780..31933f5 100644 --- a/startup/user_collection/user_HighVoltage.py +++ b/startup/user_collection/user_HighVoltage.py @@ -789,7 +789,7 @@ def waxs_on_inner(): # for inner-outer stitching # cms.SAXS.setCalibration([748, 1680-590], 2.01, [-64, -73]) # 13.5 keV cms.SAXS.setCalibration([761, 1680 - 606], 5.0, [-65, -73]) -RE.md["experiment_alias_directory"] = "/nsls2/data/cms/legacy/xf11bm/data/2022_1/LZhu2/" +RE.md["experiment_alias_directory"] = proposal_path() + "experiments/LZhu2/" if True: cali = CapillaryHolder(base=stg) diff --git a/startup/user_collection/user_LinkamTensile.py b/startup/user_collection/user_LinkamTensile.py index 3a8b173..08aed26 100644 --- a/startup/user_collection/user_LinkamTensile.py +++ b/startup/user_collection/user_LinkamTensile.py @@ -39,7 +39,7 @@ # RE.install_suspender(sus) -RE.md["experiment_alias_directory"] = "/nsls2/data/cms/legacy/xf11bm/data/2023_1/beamline" +RE.md["experiment_alias_directory"] = proposal_path() + "experiments/beamline" # cms.SAXS.setCalibration([737, 1680-582], 3, [-65, -73]) #3m, 13.5kev # cms.SAXS.setCalibration([738, 1097], 3.0, [-65, -73]) #3m,13.5kev # cms.SAXS.setCalibration([738, 1680-590], 2, [-65, -73]) diff --git a/startup/user_collection/user_LinkamThermal.py b/startup/user_collection/user_LinkamThermal.py index d5c7cbc..c3f570d 100644 --- a/startup/user_collection/user_LinkamThermal.py +++ b/startup/user_collection/user_LinkamThermal.py @@ -25,7 +25,7 @@ # RE.install_suspender(sus) -RE.md["experiment_alias_directory"] = "/nsls2/data/cms/legacy/xf11bm/data/2023_1/beamline/Commissioning" +RE.md["experiment_alias_directory"] = proposal_path() + "experiments/beamline/Commissioning" # cms.SAXS.setCalibration([737, 1680-582], 3, [-65, -73]) #3m, 13.5kev # cms.SAXS.setCalibration([738, 1097], 3.0, [-65, -73]) #3m,13.5kev # cms.SAXS.setCalibration([738, 1680-590], 2, [-65, -73]) diff --git a/startup/user_collection/user_TSAXSWAXS_stitching.py b/startup/user_collection/user_TSAXSWAXS_stitching.py index 6ef33d0..ea7cb57 100755 --- a/startup/user_collection/user_TSAXSWAXS_stitching.py +++ b/startup/user_collection/user_TSAXSWAXS_stitching.py @@ -787,7 +787,7 @@ def waxs_on_inner(): # for inner-outer stitching # cms.SAXS.setCalibration([748, 1680-590], 2.01, [-64, -73]) # 13.5 keV cms.SAXS.setCalibration([761, 1680 - 606], 5.0, [-65, -73]) -RE.md["experiment_alias_directory"] = "/nsls2/data/cms/legacy/xf11bm/data/2022_1/LZhu/" +RE.md["experiment_alias_directory"] = proposal_path() + "experiments/LZhu/" if True: cali = CapillaryHolder(base=stg) diff --git a/startup/user_collection/user_openMAXS.py.LZhu2022C1 b/startup/user_collection/user_openMAXS.py.LZhu2022C1 index eed9dea..8c9cf0c 100644 --- a/startup/user_collection/user_openMAXS.py.LZhu2022C1 +++ b/startup/user_collection/user_openMAXS.py.LZhu2022C1 @@ -766,7 +766,7 @@ def waxs_on_inner(): #for inner-outer stitching #cms.SAXS.setCalibration([748, 1680-590], 2.01, [-64, -73]) # 13.5 keV cms.SAXS.setCalibration([761, 1680-606], 5.8, [-65, -73]) -RE.md['experiment_alias_directory'] = '/nsls2/data/cms/legacy/xf11bm/data/2022_1/LZhu3/' +RE.md['experiment_alias_directory'] = proposal_path() + "experiments/LZhu3/" if True: From c5915c77032528747490b83f3e6c94d9bacaaf14 Mon Sep 17 00:00:00 2001 From: CMS Operator Date: Fri, 16 May 2025 17:17:12 -0400 Subject: [PATCH 07/15] Prefect testing --- .gitignore | 1 + startup/02-data_security.py | 3 +++ 2 files changed, 4 insertions(+) diff --git a/.gitignore b/.gitignore index 5e04c34..f0f67c5 100755 --- a/.gitignore +++ b/.gitignore @@ -139,3 +139,4 @@ doc/source/_generated_images/* # Backups *.pybak +.ipynb_checkpoints/ \ No newline at end of file diff --git a/startup/02-data_security.py b/startup/02-data_security.py index 3b64749..4fb2bb0 100644 --- a/startup/02-data_security.py +++ b/startup/02-data_security.py @@ -7,3 +7,6 @@ RE.md.pop('experiment_project', None) RE.md.pop('experiment_group', None) RE.md.pop('experiment_cycle', None) + +RE.md["experiment_alias_directory"] = "data_security_test" #NOTE TO SELF: Don't forget to delte this +RE.md['savename'] = "testname" \ No newline at end of file From 4c8a9d6650c0745c7bae7726ae905136043ba7bd Mon Sep 17 00:00:00 2001 From: CMS Operator Date: Mon, 28 Jul 2025 10:08:34 -0400 Subject: [PATCH 08/15] MNT: modified gitignore --- .gitignore | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index f0f67c5..aea5894 100755 --- a/.gitignore +++ b/.gitignore @@ -139,4 +139,8 @@ doc/source/_generated_images/* # Backups *.pybak -.ipynb_checkpoints/ \ No newline at end of file +.ipynb_checkpoints/ +*.visionenv + +# Audio Files +*.wav \ No newline at end of file From ada43007e9a86a8fe1dee9d0973583ceb35812f4 Mon Sep 17 00:00:00 2001 From: CMS Operator Date: Mon, 28 Jul 2025 15:41:55 -0400 Subject: [PATCH 09/15] ENH: Data Security 07/28/2025 --- startup/.cms_config | 3 +- .../94-sample-checkpoint.py | 104 +++++++++--------- startup/00-startup.py | 9 +- startup/20-area-detectors.py | 13 ++- startup/43-endstation-ioLogik.py | 2 + startup/81-beam.py | 16 +-- startup/82-beamstop.py | 8 +- startup/90-bluesky.py | 2 +- startup/94-sample.py | 89 ++++++++------- startup/94-sample.py.save | 26 ++--- startup/95-sample-custom.py | 2 +- startup/992-PTA-mobile.py | 2 +- startup/beamstop_config.cfg | 8 ++ startup/user_collection/user_LinkamThermal.py | 6 +- 14 files changed, 159 insertions(+), 131 deletions(-) diff --git a/startup/.cms_config b/startup/.cms_config index a967cfe..6a7ca25 100644 --- a/startup/.cms_config +++ b/startup/.cms_config @@ -1231,4 +1231,5 @@ 1229,44.45,62.8,5.0,4.0,"[-96.3, -198.0, -122.5, -1.0]","[0.0, -101.80002499999999, -93.79981875, 89.2]","[0.0, -104.9, 0.0, 90.0]","[-97.5121875, -100.80002499999999, -93.79981875, 89.2]","[51.5, -1.87]","[-30.0, -2.37]",,,-17.701,Mon May 12 02:42:31 2025 1230,44.45,62.8,5.0,4.0,"[-96.3, -198.0, -122.5, -1.0]","[0.0, -101.80002499999999, -93.79981875, 89.2]","[0.0, -104.9, 0.0, 90.0]","[-97.5121875, -100.80002499999999, -93.79981875, 89.2]","[51.5, -1.87]","[-30.0, -2.37]",,,-17.701,Tue May 13 21:47:30 2025 1231,44.45,62.8,5.0,4.0,"[-96.3, -198.0, -122.5, -1.0]","[0.0, -101.80002499999999, -93.79981875, 89.2]","[0.0, -104.9, 0.0, 90.0]","[-97.5121875, -100.80002499999999, -93.79981875, 89.2]","[51.5, -1.87]","[-30.0, -2.37]",,,-15.600142,Tue May 13 21:55:17 2025 -1232,44.45,62.8,5.0,4.0,"[-96.3, -198.0, -122.5, -1.0]","[0.0, -101.80002499999999, -93.79981875, 89.2]","[0.0, -104.9, 0.0, 90.0]","[-97.5121875, -100.80002499999999, -93.79981875, 89.2]","[51.5, -1.87]","[-30.0, -2.37]",,,-15.600674999999999,Tue May 13 21:56:00 2025 +1232,44.45,62.8,5.0,4.0,"[-96.3, -198.0, -122.5, -1.0]","[0.0, -101.80002499999999, -93.79981875, 89.2]","[0.0, -104.9, 0.0, 90.0]","[-97.5121875, -100.80002499999999, -93.79981875, 89.2]","[51.5, -1.87]","[-30.0, -2.37]",,,-15.600675,Tue May 13 21:56:00 2025 +1233,44.45,62.8,5.0,4.0,"[-96.3, -198.0, -122.5, -1.0]","[0.0, -101.80002499999999, -93.79981875, 89.2]","[0.0, -104.9, 0.0, 90.0]","[-97.5121875, -100.80002499999999, -93.79981875, 89.2]","[51.5, -1.87]","[-30.0, -2.37]",,-1.7,-17.900453,Mon Jul 28 11:38:49 2025 diff --git a/startup/.ipynb_checkpoints/94-sample-checkpoint.py b/startup/.ipynb_checkpoints/94-sample-checkpoint.py index 8206e4c..15677a2 100644 --- a/startup/.ipynb_checkpoints/94-sample-checkpoint.py +++ b/startup/.ipynb_checkpoints/94-sample-checkpoint.py @@ -1746,9 +1746,9 @@ def get_measurement_md(self, prefix=None, **md): # md_current['detector_sequence_ID'] = caget('XF:11BMB-ES{Det:SAXS}:cam1:FileNumber_RBV') # md_current['detector_sequence_ID'] = caget('XF:11BMB-ES{}:cam1:FileNumber_RBV'.format(pilatus_Epicsname)) - if get_beamline().detector[0].name is "pilatus300": + if get_beamline().detector[0].name is "pilatus300k-1": md_current["detector_sequence_ID"] = caget("XF:11BMB-ES{Det:SAXS}:cam1:FileNumber_RBV") - elif get_beamline().detector[0].name is "pilatus2M": + elif get_beamline().detector[0].name is "pilatus2m-1": md_current["detector_sequence_ID"] = caget("XF:11BMB-ES{Det:PIL2M}:cam1:FileNumber_RBV") md_current.update(get_beamline().get_md()) @@ -1778,10 +1778,10 @@ def _expose_manual(self, exposure_time=None, verbosity=3, poling_period=0.1, **m # caput('XF:11BMB-ES{}:cam1:AcquireTime'.format(pilatus_Epicsname), exposure_time) # caput('XF:11BMB-ES{}:cam1:AcquirePeriod'.format(pilatus_Epicsname), exposure_time+0.1) - if get_beamline().detector[0].name is "pilatus300": + if get_beamline().detector[0].name is "pilatus300k-1": caput("XF:11BMB-ES{Det:SAXS}:cam1:AcquireTime", exposure_time) caput("XF:11BMB-ES{Det:SAXS}:cam1:AcquirePeriod", exposure_time + 0.1) - elif get_beamline().detector[0].name is "pilatus2M": + elif get_beamline().detector[0].name is "pilatus2m-1": caput("XF:11BMB-ES{Det:PIL2M}:cam1:AcquireTime", exposure_time) caput("XF:11BMB-ES{Det:PIL2M}:cam1:AcquirePeriod", exposure_time + 0.1) @@ -1826,9 +1826,9 @@ def expose(self, exposure_time=None, extra=None, handlefile=True, verbosity=3, p exposure_time != detector.cam.acquire_time.get() ): # caget('XF:11BMB-ES{Det:PIL2M}:cam1:AcquireTime'): RE(detector.setExposureTime(exposure_time, verbosity=verbosity)) - # if detector.name is 'pilatus800' and exposure_time != detector.cam.acquire_time.get(): #caget('XF:11BMB-ES{Det:PIL2M}:cam1:AcquireTime'): + # if detector.name is "pilatus800k-1" and exposure_time != detector.cam.acquire_time.get(): #caget('XF:11BMB-ES{Det:PIL2M}:cam1:AcquireTime'): # RE(detector.setExposureTime(exposure_time, verbosity=verbosity)) - # if detector.name is 'pilatus300' and exposure_time != detector.cam.acquire_time.get(): + # if detector.name is "pilatus300k-1" and exposure_time != detector.cam.acquire_time.get(): # detector.setExposureTime(exposure_time, verbosity=verbosity) ##extra wait time when changing the exposure time. ##time.sleep(2) @@ -1864,23 +1864,23 @@ def expose(self, exposure_time=None, extra=None, handlefile=True, verbosity=3, p # Wait for detectors to be ready max_exposure_time = 0.1 for detector in get_beamline().detector: - if detector.name is "pilatus300": + if detector.name is "pilatus300k-1": current_exposure_time = detector.cam.acquire_time.get() max_exposure_time = max(max_exposure_time, current_exposure_time) - elif detector.name is "pilatus2M": + elif detector.name is "pilatus2m-1": current_exposure_time = detector.cam.acquire_time.get() max_exposure_time = max(max_exposure_time, current_exposure_time) - elif detector.name is "pilatus800" or detector.name is "pilatus8002": + elif detector.name is "pilatus800k-1" or detector.name is "pilatus800k-2": current_exposure_time = detector.cam.acquire_time.get() max_exposure_time = max(max_exposure_time, current_exposure_time) - # if detector.name is 'pilatus300': + # if detector.name is "pilatus300k-1": # current_exposure_time = caget('XF:11BMB-ES{Det:SAXS}:cam1:AcquireTime') # max_exposure_time = max(max_exposure_time, current_exposure_time) - # elif detector.name is 'pilatus2M': + # elif detector.name is "pilatus2m-1": # current_exposure_time = caget('XF:11BMB-ES{Det:PIL2M}:cam1:AcquireTime') # max_exposure_time = max(max_exposure_time, current_exposure_time) - # elif detector.name is 'pilatus800': + # elif detector.name is "pilatus800k-1": # current_exposure_time = caget('XF:11BMB-ES{Det:PIL800K}:cam1:AcquireTime') # max_exposure_time = max(max_exposure_time, current_exposure_time) # elif detector.name is 'PhotonicSciences_CMS': @@ -1927,13 +1927,13 @@ def expose(self, exposure_time=None, extra=None, handlefile=True, verbosity=3, p # Wait for detectors to be ready max_exposure_time = 0.1 for detector in get_beamline().detector: - if detector.name is "pilatus300": + if detector.name is "pilatus300k-1": current_exposure_time = detector.cam.acquire_time.get() max_exposure_time = max(max_exposure_time, current_exposure_time) - elif detector.name is "pilatus2M": + elif detector.name is "pilatus2m-1": current_exposure_time = detector.cam.acquire_time.get() max_exposure_time = max(max_exposure_time, current_exposure_time) - elif detector.name is "pilatus800" or detector.name is "pilatus8002": + elif detector.name is "pilatus800k-1" or detector.name is "pilatus800k-2": current_exposure_time = detector.cam.acquire_time.get() max_exposure_time = max(max_exposure_time, current_exposure_time) @@ -1967,9 +1967,9 @@ def _expose_test(self, exposure_time=None, extra=None, handlefile=True, verbosit exposure_time != detector.cam.acquire_time.get() ): # caget('XF:11BMB-ES{Det:PIL2M}:cam1:AcquireTime'): RE(detector.setExposureTime(exposure_time, verbosity=verbosity)) - # if detector.name is 'pilatus800' and exposure_time != detector.cam.acquire_time.get(): #caget('XF:11BMB-ES{Det:PIL2M}:cam1:AcquireTime'): + # if detector.name is "pilatus800k-1" and exposure_time != detector.cam.acquire_time.get(): #caget('XF:11BMB-ES{Det:PIL2M}:cam1:AcquireTime'): # RE(detector.setExposureTime(exposure_time, verbosity=verbosity)) - # if detector.name is 'pilatus300' and exposure_time != detector.cam.acquire_time.get(): + # if detector.name is "pilatus300k-1" and exposure_time != detector.cam.acquire_time.get(): # detector.setExposureTime(exposure_time, verbosity=verbosity) ##extra wait time when changing the exposure time. ##time.sleep(2) @@ -2009,13 +2009,13 @@ def _expose_test(self, exposure_time=None, extra=None, handlefile=True, verbosit # Wait for detectors to be ready max_exposure_time = 0.1 for detector in get_beamline().detector: - if detector.name is "pilatus300": + if detector.name is "pilatus300k-1": current_exposure_time = caget("XF:11BMB-ES{Det:SAXS}:cam1:AcquireTime") max_exposure_time = max(max_exposure_time, current_exposure_time) - elif detector.name is "pilatus2M": + elif detector.name is "pilatus2m-1": current_exposure_time = caget("XF:11BMB-ES{Det:PIL2M}:cam1:AcquireTime") max_exposure_time = max(max_exposure_time, current_exposure_time) - elif detector.name is "pilatus800": + elif detector.name is "pilatus800k-1": current_exposure_time = caget("XF:11BMB-ES{Det:PIL800K}:cam1:AcquireTime") max_exposure_time = max(max_exposure_time, current_exposure_time) # elif detector.name is 'PhotonicSciences_CMS': @@ -2038,13 +2038,13 @@ def _expose_test(self, exposure_time=None, extra=None, handlefile=True, verbosit status = 1 for detector in get_beamline().detector: - if detector.name is "pilatus300": + if detector.name is "pilatus300k-1": if caget("XF:11BMB-ES{Det:SAXS}:cam1:Acquire") == 1: status *= 0 - elif detector.name is "pilatus2M": + elif detector.name is "pilatus2m-1": if caget("XF:11BMB-ES{Det:PIL2M}:cam1:Acquire") == 1: status *= 0 - elif detector.name is "pilatus800": + elif detector.name is "pilatus800k-1": if caget("XF:11BMB-ES{Det:PIL800K}:cam1:Acquire") == 1: status *= 0 # elif detector.name is 'PhotonicSciences_CMS': @@ -2073,13 +2073,13 @@ def _expose_test(self, exposure_time=None, extra=None, handlefile=True, verbosit def handle_file(self, detector, extra=None, verbosity=3, subdirs=True, linksave=True, **md): subdir = "" if subdirs: - if detector.name is "pilatus300" or detector.name is "pilatus8002": + if detector.name is "pilatus300k-1" or detector.name is "pilatus800k-2": subdir = "/maxs/raw/" detname = "maxs" - elif detector.name is "pilatus2M": + elif detector.name is "pilatus2m-1": subdir = "/saxs/raw/" detname = "saxs" - elif detector.name is "pilatus800": + elif detector.name is "pilatus800k-1": subdir = "/waxs/raw/" detname = "waxs" else: @@ -2125,7 +2125,7 @@ def handle_file(self, detector, extra=None, verbosity=3, subdirs=True, linksave= def _old_handle_file(self, detector, extra=None, verbosity=3, subdirs=True, linksave=True, **md): subdir = "" - if detector.name is "pilatus300" or detector.name is "pilatus8002": + if detector.name is "pilatus300k-1" or detector.name is "pilatus800k-2": # chars = caget('XF:11BMB-ES{Det:SAXS}:TIFF1:FullFileName_RBV') # filename = ''.join(chr(char) for char in chars)[:-1] filename = detector.tiff.full_file_name.get() # RL, 20210831 @@ -2165,7 +2165,7 @@ def _old_handle_file(self, detector, extra=None, verbosity=3, subdirs=True, link if verbosity >= 3: print(" Data linked as: {}".format(link_name)) - elif detector.name is "pilatus2M": + elif detector.name is "pilatus2m-1": foldername = "/nsls2/xf11bm/" # chars = caget('XF:11BMB-ES{Det:PIL2M}:TIFF1:FullFileName_RBV') @@ -2212,7 +2212,7 @@ def _old_handle_file(self, detector, extra=None, verbosity=3, subdirs=True, link if verbosity >= 3: print(" Data linked as: {}".format(link_name)) - elif detector.name is "pilatus800": + elif detector.name is "pilatus800k-1": foldername = "/nsls2/xf11bm/" # chars = caget('XF:11BMB-ES{Det:PIL800K}:TIFF1:FullFileName_RBV') @@ -2763,7 +2763,7 @@ def measureRock( exposure_time != detector.cam.acquire_time.get() ): # caget('XF:11BMB-ES{Det:PIL2M}:cam1:AcquireTime'): RE(detector.setExposureTime(exposure_time, verbosity=verbosity)) - # if detector.name is 'pilatus300' and exposure_time != detector.cam.acquire_time.get(): + # if detector.name is "pilatus300k-1" and exposure_time != detector.cam.acquire_time.get(): # detector.setExposureTime(exposure_time, verbosity=verbosity) ##extra wait time when changing the exposure time. ##time.sleep(2) @@ -2800,16 +2800,16 @@ def measureRock( # Wait for detectors to be ready max_exposure_time = 0 for detector in get_beamline().detector: - if detector.name is "pilatus300" or "pilatus800" or "pilatus2M" or "pilatus8002": + if detector.name is "pilatus300k-1" or "pilatus800k-1" or "pilatus2m-1" or "pilatus800k-2": max_exposure_time = detector.cam.acquire_time.get() - # if detector.name is 'pilatus300': + # if detector.name is "pilatus300k-1": # current_exposure_time = caget('XF:11BMB-ES{Det:SAXS}:cam1:AcquireTime') # max_exposure_time = max(max_exposure_time, current_exposure_time) - # elif detector.name is 'pilatus2M': + # elif detector.name is "pilatus2m-1": # current_exposure_time = caget('XF:11BMB-ES{Det:PIL2M}:cam1:AcquireTime') # max_exposure_time = max(max_exposure_time, current_exposure_time) - # elif detector.name is 'pilatus800': + # elif detector.name is "pilatus800k-1": # current_exposure_time = caget('XF:11BMB-ES{Det:PIL800K}:cam1:AcquireTime') # max_exposure_time = max(max_exposure_time, current_exposure_time) # elif detector.name is 'PhotonicSciences_CMS': @@ -2834,13 +2834,13 @@ def measureRock( if detector.cam.acquire.get(): status *= 0 - # if detector.name is 'pilatus300': + # if detector.name is "pilatus300k-1": # if caget('XF:11BMB-ES{Det:SAXS}:cam1:Acquire')==1: # status *= 0 - # elif detector.name is 'pilatus2M': + # elif detector.name is "pilatus2m-1": # if caget('XF:11BMB-ES{Det:PIL2M}:cam1:Acquire')==1: # status *= 0 - # elif detector.name is 'pilatus800': + # elif detector.name is "pilatus800k-1": # if caget('XF:11BMB-ES{Det:PIL800K}:cam1:Acquire')==1: # status *= 0 # elif detector.name is 'PhotonicSciences_CMS': @@ -3015,7 +3015,7 @@ def _test_expose( # Wait for detectors to be ready max_exposure_time = 0 for detector in get_beamline().detector: - if detector.name is "pilatus300" or "pilatus2M": + if detector.name is "pilatus300k-1" or "pilatus2m-1": current_exposure_time = caget("XF:11BMB-ES{}:cam1:AcquireTime".format(pilatus_Epicsname)) max_exposure_time = max(max_exposure_time, current_exposure_time) elif detector.name is "PhotonicSciences_CMS": @@ -3044,7 +3044,7 @@ def _test_expose( status = 1 for detector in get_beamline().detector: - if detector.name is "pilatus300" or "pilatus2M": + if detector.name is "pilatus300k-1" or "pilatus2m-1": print("status2.5 = ", status) if caget("XF:11BMB-ES{}:cam1:Acquire".format(pilatus_Epicsname)) == 1: status = 0 @@ -3548,28 +3548,28 @@ def series_measure( print("handling the file names") self.handle_fileseries(detector, num_frames=num_frames, extra=extra, verbosity=verbosity, **md) - # if detector.name is 'pilatus2M': + # if detector.name is "pilatus2m-1": # caput('XF:11BMB-ES{Det:PIL2M}:cam1:NumImages', 1) - # if detector.name is 'pilatus300' : + # if detector.name is "pilatus300k-1" : # caput('XF:11BMB-ES{Det:SAXS}:cam1:NumImages', 1) - # if detector.name is 'pilatus800' : + # if detector.name is "pilatus800k-1" : # caput('XF:11BMB-ES{Det:PIL800K}:cam1:NumImages', 1) def initialDetector(self): # reset the num_frame back to 1 for detector in get_beamline().detector: detector.cam.num_images.put(1) - # if detector.name is 'pilatus2M': + # if detector.name is "pilatus2m-1": # caput('XF:11BMB-ES{Det:PIL2M}:cam1:NumImages', 1) - # if detector.name is 'pilatus300' : + # if detector.name is "pilatus300k-1" : # caput('XF:11BMB-ES{Det:SAXS}:cam1:NumImages', 1) - # if detector.name is 'pilatus800' : + # if detector.name is "pilatus800k-1" : # caput('XF:11BMB-ES{Det:PIL800K}:cam1:NumImages', 1) def _old_handle_fileseries(self, detector, num_frames=None, extra=None, verbosity=3, subdirs=True, **md): subdir = "" - if detector.name == "pilatus300" or detector.name == "pilatus8002": + if detector.name == "pilatus300k-1" or detector.name == "pilatus800k-2": # chars = caget('XF:11BMB-ES{Det:SAXS}:TIFF1:FullFileName_RBV') # filename = ''.join(chr(char) for char in chars)[:-1] # filename_part1 = ''.join(chr(char) for char in chars)[:-13] @@ -3622,7 +3622,7 @@ def _old_handle_fileseries(self, detector, num_frames=None, extra=None, verbosit if num_frame == 0 or num_frame == np.max(num_frames): print(" Data {} linked as: {}".format(filename_new, link_name_new)) - elif detector.name == "pilatus2M": + elif detector.name == "pilatus2m-1": # chars = caget('XF:11BMB-ES{Det:PIL2M}:TIFF1:FullFileName_RBV') # filename = ''.join(chr(char) for char in chars)[:-1] # filename_part1 = ''.join(chr(char) for char in chars)[:-13] @@ -3679,12 +3679,12 @@ def _old_handle_fileseries(self, detector, num_frames=None, extra=None, verbosit if num_frame == 0 or num_frame == np.max(num_frames): print(" Data {} linked as: {}".format(filename_new, link_name_new)) - # elif detector.name is 'pilatus800': + # elif detector.name is "pilatus800k-1": # chars = caget('XF:11BMB-ES{Det:PIL800K}:TIFF1:FullFileName_RBV') # filename = ''.join(chr(char) for char in chars)[:-1] # filename_part1 = ''.join(chr(char) for char in chars)[:-13] - elif detector.name == "pilatus800": + elif detector.name == "pilatus800k-1": foldername = "/nsls2/xf11bm/" # chars = caget('XF:11BMB-ES{Det:PIL800K}:TIFF1:FullFileName_RBV') @@ -3751,15 +3751,15 @@ def _old_handle_fileseries(self, detector, num_frames=None, extra=None, verbosit def handle_fileseries(self, detector, num_frames=None, extra=None, verbosity=3, subdirs=True, **md): subdir = "" if subdirs: - if detector.name == "pilatus300" or detector.name == "pilatus8002": + if detector.name == "pilatus300k-1" or detector.name == "pilatus800k-2": subdir = "/maxs/raw/" detname = "maxs" print("{} data handling".format(detector.name)) - elif detector.name == "pilatus2M": + elif detector.name == "pilatus2m-1": subdir = "/saxs/raw/" detname = "saxs" print("pilatus2M data handling") - elif detector.name == "pilatus800": + elif detector.name == "pilatus800k-1": subdir = "/waxs/raw/" detname = "waxs" print("pilatus800k data handling") diff --git a/startup/00-startup.py b/startup/00-startup.py index 4427948..0543680 100644 --- a/startup/00-startup.py +++ b/startup/00-startup.py @@ -3,6 +3,8 @@ import nslsii import os +os.environ.pop('TILED_API_KEY') + from tiled.client import from_profile from databroker import Broker from redis_json_dict import RedisJSONDict @@ -24,8 +26,8 @@ def in_prompt_tokens(self, cli=None): ip = get_ipython() ip.prompts = ProposalIDPrompt(ip) -# # Configure a Tiled writing client -# tiled_writing_client = from_profile("nsls2", api_key=os.environ["TILED_BLUESKY_WRITING_API_KEY_CMS"])["cms"]["raw"] +# Configure a Tiled writing client +tiled_writing_client = from_profile("nsls2", api_key=os.environ["TILED_BLUESKY_WRITING_API_KEY_CMS"])["cms"]["raw"] class TiledInserter: @@ -47,7 +49,6 @@ def insert(self, name, doc): raise error tiled_inserter = TiledInserter() -tiled_inserter = "cms" nslsii.configure_base(get_ipython().user_ns, tiled_inserter, @@ -55,7 +56,7 @@ def insert(self, name, doc): redis_url="info.cms.nsls2.bnl.gov") print("Initializing Tiled reading client...\nMake sure you check for duo push.") -tiled_reading_client = from_profile("nsls2", username=None, include_data_sources=True)["cms"]["raw"] +tiled_reading_client = cat = from_profile("nsls2", username=None)["cms"]["raw"] # db = Broker(tiled_reading_client) diff --git a/startup/20-area-detectors.py b/startup/20-area-detectors.py index 7f73ef9..3fe5438 100644 --- a/startup/20-area-detectors.py +++ b/startup/20-area-detectors.py @@ -151,6 +151,15 @@ def stage(self, *args, **kwargs): self.tiff.reg_root = assets_path() + f'{self.name}' return super().stage(*args, **kwargs) + def setExposureTime(self, exposure_time, verbosity=3): + yield from mv(self.cam.acquire_time, self.cam.acquire_time.get()) # noop + + def setExposurePeriod(self, exposure_period, verbosity=3): + yield from mv(self.cam.acquire_period, self.cam.acquire_period.get()) # noop + + def setExposureNumber(self, exposure_number, verbosity=3): + yield from mv(self.cam.num_images, self.cam.num_images.get()) # noop + class PilatusDetectorCamV33(PilatusDetectorCam): """This is used to update the standard prosilica to AD33.""" @@ -514,8 +523,8 @@ def stage(self): # pilatus300 section # if True: if Pilatus300_on == True: - pilatus300 = Pilatus300V33("XF:11BMB-ES{Det:SAXS}:", name="pilatus300") - # pilatus300 = PilatusV33('XF:11BMB-ES{Det:SAXS}:', name='pilatus300') + pilatus300 = Pilatus300V33("XF:11BMB-ES{Det:SAXS}:", name="pilatus300k-1") + # pilatus300 = PilatusV33('XF:11BMB-ES{Det:SAXS}:', name="pilatus300k-1") pilatus300.tiff.read_attrs = [] pilatus300.stats3.total.kind = "hinted" pilatus300.stats4.total.kind = "hinted" diff --git a/startup/43-endstation-ioLogik.py b/startup/43-endstation-ioLogik.py index 3e7fc00..c5b6b21 100644 --- a/startup/43-endstation-ioLogik.py +++ b/startup/43-endstation-ioLogik.py @@ -1,3 +1,5 @@ +print(f'Loading {__file__}') + import time from ophyd import Device diff --git a/startup/81-beam.py b/startup/81-beam.py index 785870b..234ea5f 100644 --- a/startup/81-beam.py +++ b/startup/81-beam.py @@ -3350,13 +3350,13 @@ def setDirectBeamROI(self, size=[10, 4], verbosity=3): The size is changed to [10, 4] for possible beam drift during a user run (changed at 08/16/17) """ - if pilatus_name.name == "pilatus2M": + if pilatus_name.name == "pilatus2m-1": detector = self.SAXS # These positions are updated based on current detector position det_md = detector.get_md() x0 = det_md["detector_SAXS_x0_pix"] y0 = det_md["detector_SAXS_y0_pix"] - if pilatus_name.name == "pilatus800": + if pilatus_name.name == "pilatus800k-1": detector = self.WAXS # These positions are updated based on current detector position @@ -3385,13 +3385,13 @@ def setReflectedBeamROI(self, total_angle=0.16, size=[10, 2], verbosity=3): The size argument controls the size (in pixels) of the ROI itself (in the format [width, height]). A size=[6,2] is reasonable.""" - if pilatus_name.name == "pilatus2M": + if pilatus_name.name == "pilatus2m-1": detector = self.SAXS # These positions are updated based on current detector position det_md = detector.get_md() x0 = det_md["detector_SAXS_x0_pix"] y0 = det_md["detector_SAXS_y0_pix"] - if pilatus_name.name == "pilatus800": + if pilatus_name.name == "pilatus800k-1": detector = self.WAXS # These positions are updated based on current detector position @@ -3438,13 +3438,13 @@ def setROI2ReflectBeamROI(self, total_angle=0.16, size=[10, 100], verbosity=3): The size argument controls the size (in pixels) of the ROI itself (in the format [width, height]). A size=[6,2] is reasonable.""" - if pilatus_name.name == "pilatus2M": + if pilatus_name.name == "pilatus2m-1": detector = self.SAXS # These positions are updated based on current detector position det_md = detector.get_md() x0 = det_md["detector_SAXS_x0_pix"] y0 = det_md["detector_SAXS_y0_pix"] - if pilatus_name.name == "pilatus800": + if pilatus_name.name == "pilatus800k-1": detector = self.WAXS # These positions are updated based on current detector position @@ -3521,14 +3521,14 @@ def setSpecularReflectivityROI(self, total_angle=0.16, size=[10, 10], default_SA y_offset_pix = y_offset_mm / pixel_size # for pilatus800k - if pilatus_name.name == "pilatus800": + if pilatus_name.name == "pilatus800k-1": y_pos = int(y0 - size[1] / 2 - y_offset_pix) # for pilatus2M, placed up-side down # y_pos = int( y0 - size[1]/2 + y_offset_pix ) # for pilatus2M, with pattern rotated 180deg. changed at 052918 - if pilatus_name.name == "pilatus2M": + if pilatus_name.name == "pilatus2m-1": y_pos = int(y0 - size[1] / 2 - y_offset_pix) # y pixels for intermodule gaps, for pilatus2M (195 pixels high module, 17 pixels high gap) diff --git a/startup/82-beamstop.py b/startup/82-beamstop.py index ac0c3c5..fc20475 100644 --- a/startup/82-beamstop.py +++ b/startup/82-beamstop.py @@ -5,8 +5,10 @@ from pathlib import Path from datetime import datetime +folder = '/home/xf11bm/.ipython/profile_collection/startup/' + class Beamstop: - def __init__(self, name, config_file='beamstop_config.cfg'): + def __init__(self, name, config_file = folder + 'beamstop_config.cfg'): self.name = name self.config_file = Path(config_file) self.bsx = bsx.position @@ -16,12 +18,12 @@ def __init__(self, name, config_file='beamstop_config.cfg'): self.load() @classmethod - def get(cls, name, config_file='beamstop_config.cfg'): + def get(cls, name, config_file = folder + 'beamstop_config.cfg'): print(f"Set current beamstop to '{name}' without moving.") return cls(name, config_file=config_file) @classmethod - def goto(cls, name, config_file='beamstop_config.cfg'): + def goto(cls, name, config_file = folder + 'beamstop_config.cfg'): bs = cls(name, config_file=config_file) # bs._move() RE(bs._move()) diff --git a/startup/90-bluesky.py b/startup/90-bluesky.py index d6c3728..d23c338 100644 --- a/startup/90-bluesky.py +++ b/startup/90-bluesky.py @@ -369,7 +369,7 @@ def config_update(): current_config = { "bsx_pos": cms.bsx_pos, - #'armr_absorber_o':beam.armr_absorber_o, + 'armr_absorber_o':beam.armr_absorber_o, "_delta_y_hover": robot._delta_y_hover, "_delta_y_slot": robot._delta_y_slot, "_delta_garage_x": robot._delta_garage_x, diff --git a/startup/94-sample.py b/startup/94-sample.py index 5a21263..dc70300 100644 --- a/startup/94-sample.py +++ b/startup/94-sample.py @@ -1782,9 +1782,9 @@ def get_measurement_md(self, prefix=None, **md): # md_current['detector_sequence_ID'] = caget('XF:11BMB-ES{Det:SAXS}:cam1:FileNumber_RBV') # md_current['detector_sequence_ID'] = caget('XF:11BMB-ES{}:cam1:FileNumber_RBV'.format(pilatus_Epicsname)) - if get_beamline().detector[0].name == "pilatus300": + if get_beamline().detector[0].name == "pilatus300k-1": md_current["detector_sequence_ID"] = caget("XF:11BMB-ES{Det:SAXS}:cam1:FileNumber_RBV") - elif get_beamline().detector[0].name == "pilatus2M": + elif get_beamline().detector[0].name == "pilatus2m-1": md_current["detector_sequence_ID"] = caget("XF:11BMB-ES{Det:PIL2M}:cam1:FileNumber_RBV") md_current.update(get_beamline().get_md()) @@ -1814,10 +1814,10 @@ def _expose_manual(self, exposure_time=None, verbosity=3, poling_period=0.1, **m # caput('XF:11BMB-ES{}:cam1:AcquireTime'.format(pilatus_Epicsname), exposure_time) # caput('XF:11BMB-ES{}:cam1:AcquirePeriod'.format(pilatus_Epicsname), exposure_time+0.1) - if get_beamline().detector[0].name == "pilatus300": + if get_beamline().detector[0].name == "pilatus300k-1": caput("XF:11BMB-ES{Det:SAXS}:cam1:AcquireTime", exposure_time) caput("XF:11BMB-ES{Det:SAXS}:cam1:AcquirePeriod", exposure_time + 0.1) - elif get_beamline().detector[0].name == "pilatus2M": + elif get_beamline().detector[0].name == "pilatus2m-1": caput("XF:11BMB-ES{Det:PIL2M}:cam1:AcquireTime", exposure_time) caput("XF:11BMB-ES{Det:PIL2M}:cam1:AcquirePeriod", exposure_time + 0.1) @@ -1862,9 +1862,9 @@ def expose(self, exposure_time=None, extra=None, handlefile=True, verbosity=3, p exposure_time != detector.cam.acquire_time.get() ): # caget('XF:11BMB-ES{Det:PIL2M}:cam1:AcquireTime'): RE(detector.setExposureTime(exposure_time, verbosity=verbosity)) - # if detector.name is 'pilatus800' and exposure_time != detector.cam.acquire_time.get(): #caget('XF:11BMB-ES{Det:PIL2M}:cam1:AcquireTime'): + # if detector.name is "pilatus800k-1" and exposure_time != detector.cam.acquire_time.get(): #caget('XF:11BMB-ES{Det:PIL2M}:cam1:AcquireTime'): # RE(detector.setExposureTime(exposure_time, verbosity=verbosity)) - # if detector.name is 'pilatus300' and exposure_time != detector.cam.acquire_time.get(): + # if detector.name is "pilatus300k-1" and exposure_time != detector.cam.acquire_time.get(): # detector.setExposureTime(exposure_time, verbosity=verbosity) ##extra wait time when changing the exposure time. ##time.sleep(2) @@ -1900,23 +1900,28 @@ def expose(self, exposure_time=None, extra=None, handlefile=True, verbosity=3, p # Wait for detectors to be ready max_exposure_time = 0.1 for detector in get_beamline().detector: - if detector.name == "pilatus300": + print('here in expose:', detector.name) + if detector.name == "pilatus300k-1": current_exposure_time = detector.cam.acquire_time.get() max_exposure_time = max(max_exposure_time, current_exposure_time) - elif detector.name == "pilatus2M": + elif detector.name == "pilatus2m-1": current_exposure_time = detector.cam.acquire_time.get() max_exposure_time = max(max_exposure_time, current_exposure_time) - elif detector.name == "pilatus800" or detector.name == "pilatus8002": + elif detector.name == "pilatus800k-1" or detector.name == "pilatus800k-2": current_exposure_time = detector.cam.acquire_time.get() max_exposure_time = max(max_exposure_time, current_exposure_time) - # if detector.name is 'pilatus300': + elif 'webcam' in detector.name: + current_exposure_time = detector.cam.acquire_time.get() + max_exposure_time = max(max_exposure_time, current_exposure_time) + + # if detector.name is "pilatus300k-1": # current_exposure_time = caget('XF:11BMB-ES{Det:SAXS}:cam1:AcquireTime') # max_exposure_time = max(max_exposure_time, current_exposure_time) - # elif detector.name is 'pilatus2M': + # elif detector.name is "pilatus2m-1": # current_exposure_time = caget('XF:11BMB-ES{Det:PIL2M}:cam1:AcquireTime') # max_exposure_time = max(max_exposure_time, current_exposure_time) - # elif detector.name is 'pilatus800': + # elif detector.name is "pilatus800k-1": # current_exposure_time = caget('XF:11BMB-ES{Det:PIL800K}:cam1:AcquireTime') # max_exposure_time = max(max_exposure_time, current_exposure_time) # elif detector.name is 'PhotonicSciences_CMS': @@ -1963,26 +1968,26 @@ def expose(self, exposure_time=None, extra=None, handlefile=True, verbosity=3, p # Wait for detectors to be ready max_exposure_time = 0.1 for detector in get_beamline().detector: - if detector.name == "pilatus300": + if detector.name == "pilatus300k-1": current_exposure_time = detector.cam.acquire_time.get() max_exposure_time = max(max_exposure_time, current_exposure_time) - elif detector.name == "pilatus2M": + elif detector.name == "pilatus2m-1": current_exposure_time = detector.cam.acquire_time.get() max_exposure_time = max(max_exposure_time, current_exposure_time) - elif detector.name == "pilatus800" or detector.name == "pilatus8002": + elif detector.name == "pilatus800k-1" or detector.name == "pilatus800k-2": current_exposure_time = detector.cam.acquire_time.get() max_exposure_time = max(max_exposure_time, current_exposure_time) percentage = 100 * (time.time() - start_time) / max_exposure_time print("After re-exposing .... percentage = {} ".format(percentage)) - # if detector.name is 'pilatus300': + # if detector.name is "pilatus300k-1": # if caget('XF:11BMB-ES{Det:SAXS}:cam1:Acquire')==1: # status *= 0 - # elif detector.name is 'pilatus2M': + # elif detector.name is "pilatus2m-1": # if caget('XF:11BMB-ES{Det:PIL2M}:cam1:Acquire')==1: # status *= 0 - # elif detector.name is 'pilatus800': + # elif detector.name is "pilatus800k-1": # if caget('XF:11BMB-ES{Det:PIL800K}:cam1:Acquire')==1: # status *= 0 # elif detector.name is 'PhotonicSciences_CMS': @@ -2024,9 +2029,9 @@ def _expose_test(self, exposure_time=None, extra=None, handlefile=True, verbosit exposure_time != detector.cam.acquire_time.get() ): # caget('XF:11BMB-ES{Det:PIL2M}:cam1:AcquireTime'): RE(detector.setExposureTime(exposure_time, verbosity=verbosity)) - # if detector.name is 'pilatus800' and exposure_time != detector.cam.acquire_time.get(): #caget('XF:11BMB-ES{Det:PIL2M}:cam1:AcquireTime'): + # if detector.name is "pilatus800k-1" and exposure_time != detector.cam.acquire_time.get(): #caget('XF:11BMB-ES{Det:PIL2M}:cam1:AcquireTime'): # RE(detector.setExposureTime(exposure_time, verbosity=verbosity)) - # if detector.name is 'pilatus300' and exposure_time != detector.cam.acquire_time.get(): + # if detector.name is "pilatus300k-1" and exposure_time != detector.cam.acquire_time.get(): # detector.setExposureTime(exposure_time, verbosity=verbosity) ##extra wait time when changing the exposure time. ##time.sleep(2) @@ -2066,13 +2071,13 @@ def _expose_test(self, exposure_time=None, extra=None, handlefile=True, verbosit # Wait for detectors to be ready max_exposure_time = 0.1 for detector in get_beamline().detector: - if detector.name == "pilatus300": + if detector.name == "pilatus300k-1": current_exposure_time = caget("XF:11BMB-ES{Det:SAXS}:cam1:AcquireTime") max_exposure_time = max(max_exposure_time, current_exposure_time) - elif detector.name == "pilatus2M": + elif detector.name == "pilatus2m-1": current_exposure_time = caget("XF:11BMB-ES{Det:PIL2M}:cam1:AcquireTime") max_exposure_time = max(max_exposure_time, current_exposure_time) - elif detector.name == "pilatus800": + elif detector.name == "pilatus800k-1": current_exposure_time = caget("XF:11BMB-ES{Det:PIL800K}:cam1:AcquireTime") max_exposure_time = max(max_exposure_time, current_exposure_time) # elif detector.name is 'PhotonicSciences_CMS': @@ -2095,13 +2100,13 @@ def _expose_test(self, exposure_time=None, extra=None, handlefile=True, verbosit status = 1 for detector in get_beamline().detector: - if detector.name == "pilatus300": + if detector.name == "pilatus300k-1": if caget("XF:11BMB-ES{Det:SAXS}:cam1:Acquire") == 1: status *= 0 - elif detector.name == "pilatus2M": + elif detector.name == "pilatus2m-1": if caget("XF:11BMB-ES{Det:PIL2M}:cam1:Acquire") == 1: status *= 0 - elif detector.name == "pilatus800": + elif detector.name == "pilatus800k-1": if caget("XF:11BMB-ES{Det:PIL800K}:cam1:Acquire") == 1: status *= 0 # elif detector.name is 'PhotonicSciences_CMS': @@ -2130,13 +2135,13 @@ def _expose_test(self, exposure_time=None, extra=None, handlefile=True, verbosit def handle_file(self, detector, extra=None, verbosity=3, subdirs=True, linksave=True, **md): subdir = "" if subdirs: - if detector.name == "pilatus300" or detector.name == "pilatus8002": + if detector.name == "pilatus300k-1" or detector.name == "pilatus800k-2": subdir = "/maxs/raw/" detname = "maxs" - elif detector.name == "pilatus2M": + elif detector.name == "pilatus2m-1": subdir = "/saxs/raw/" detname = "saxs" - elif detector.name == "pilatus800": + elif detector.name == "pilatus800k-1": subdir = "/waxs/raw/" detname = "waxs" else: @@ -2191,7 +2196,7 @@ def handle_file(self, detector, extra=None, verbosity=3, subdirs=True, linksave= def _old_handle_file(self, detector, extra=None, verbosity=3, subdirs=True, linksave=True, **md): subdir = "" - if detector.name == "pilatus300" or detector.name == "pilatus8002": + if detector.name == "pilatus300k-1" or detector.name == "pilatus800k-2": # chars = caget('XF:11BMB-ES{Det:SAXS}:TIFF1:FullFileName_RBV') # filename = ''.join(chr(char) for char in chars)[:-1] filename = detector.tiff.full_file_name.get() # RL, 20210831 @@ -2231,7 +2236,7 @@ def _old_handle_file(self, detector, extra=None, verbosity=3, subdirs=True, link if verbosity >= 3: print(" Data linked as: {}".format(link_name)) - elif detector.name == "pilatus2M": + elif detector.name == "pilatus2m-1": foldername = "/nsls2/xf11bm/" # chars = caget('XF:11BMB-ES{Det:PIL2M}:TIFF1:FullFileName_RBV') @@ -2278,7 +2283,7 @@ def _old_handle_file(self, detector, extra=None, verbosity=3, subdirs=True, link if verbosity >= 3: print(" Data linked as: {}".format(link_name)) - elif detector.name == "pilatus800": + elif detector.name == "pilatus800k-1": foldername = "/nsls2/xf11bm/" # chars = caget('XF:11BMB-ES{Det:PIL800K}:TIFF1:FullFileName_RBV') @@ -2851,7 +2856,7 @@ def measureRock( # Wait for detectors to be ready max_exposure_time = 0 for detector in get_beamline().detector: - if detector.name in {"pilatus300", "pilatus800", "pilatus2M", "pilatus8002"}: + if detector.name in {"pilatus300k-1", "pilatus800k-1", "pilatus2m-1", "pilatus800k-2"}: max_exposure_time = detector.cam.acquire_time.get() else: if verbosity >= 1: @@ -3016,7 +3021,7 @@ def _test_expose( # Wait for detectors to be ready max_exposure_time = 0 for detector in get_beamline().detector: - if detector.name in {"pilatus300", "pilatus2M"}: + if detector.name in {"pilatus300k-1", "pilatus2m-1"}: current_exposure_time = caget("XF:11BMB-ES{}:cam1:AcquireTime".format(pilatus_Epicsname)) max_exposure_time = max(max_exposure_time, current_exposure_time) elif detector.name == "PhotonicSciences_CMS": @@ -3045,7 +3050,7 @@ def _test_expose( status = 1 for detector in get_beamline().detector: - if detector.name == "pilatus300" or "pilatus2M": + if detector.name == "pilatus300k-1" or "pilatus2m-1": print("status2.5 = ", status) if caget("XF:11BMB-ES{}:cam1:Acquire".format(pilatus_Epicsname)) == 1: status = 0 @@ -3524,7 +3529,7 @@ def initialDetector(self): def _old_handle_fileseries(self, detector, num_frames=None, extra=None, verbosity=3, subdirs=True, **md): subdir = "" - if detector.name == "pilatus300" or detector.name == "pilatus8002": + if detector.name == "pilatus300k-1" or detector.name == "pilatus800k-2": filename = detector.tiff.full_file_name.get() # RL, 20210831 print("pilatus300k data handling") @@ -3573,7 +3578,7 @@ def _old_handle_fileseries(self, detector, num_frames=None, extra=None, verbosit if num_frame == 0 or num_frame == np.max(num_frames): print(" Data {} linked as: {}".format(filename_new, link_name_new)) - elif detector.name == "pilatus2M": + elif detector.name == "pilatus2m-1": # chars = caget('XF:11BMB-ES{Det:PIL2M}:TIFF1:FullFileName_RBV') # filename = ''.join(chr(char) for char in chars)[:-1] # filename_part1 = ''.join(chr(char) for char in chars)[:-13] @@ -3630,12 +3635,12 @@ def _old_handle_fileseries(self, detector, num_frames=None, extra=None, verbosit if num_frame == 0 or num_frame == np.max(num_frames): print(" Data {} linked as: {}".format(filename_new, link_name_new)) - # elif detector.name is 'pilatus800': + # elif detector.name is "pilatus800k-1": # chars = caget('XF:11BMB-ES{Det:PIL800K}:TIFF1:FullFileName_RBV') # filename = ''.join(chr(char) for char in chars)[:-1] # filename_part1 = ''.join(chr(char) for char in chars)[:-13] - elif detector.name == "pilatus800": + elif detector.name == "pilatus800k-1": foldername = "/nsls2/xf11bm/" # chars = caget('XF:11BMB-ES{Det:PIL800K}:TIFF1:FullFileName_RBV') @@ -3702,15 +3707,15 @@ def _old_handle_fileseries(self, detector, num_frames=None, extra=None, verbosit def handle_fileseries(self, detector, num_frames=None, extra=None, verbosity=3, subdirs=True, **md): subdir = "" if subdirs: - if detector.name == "pilatus300" or detector.name == "pilatus8002": + if detector.name == "pilatus300k-1" or detector.name == "pilatus800k-2": subdir = "/maxs/raw/" detname = "maxs" print("{} data handling".format(detector.name)) - elif detector.name == "pilatus2M": + elif detector.name == "pilatus2m-1": subdir = "/saxs/raw/" detname = "saxs" print("pilatus2M data handling") - elif detector.name == "pilatus800": + elif detector.name == "pilatus800k-1": subdir = "/waxs/raw/" detname = "waxs" print("pilatus800k data handling") diff --git a/startup/94-sample.py.save b/startup/94-sample.py.save index e8f62b6..cf515c2 100755 --- a/startup/94-sample.py.save +++ b/startup/94-sample.py.save @@ -1294,9 +1294,9 @@ class Sample_Generic(CoordinateSystem): #md_current['detector_sequence_ID'] = caget('XF:11BMB-ES{Det:SAXS}:cam1:FileNumber_RBV') #md_current['detector_sequence_ID'] = caget('XF:11BMB-ES{}:cam1:FileNumber_RBV'.format(pilatus_Epicsname)) - if get_beamline().detector[0].name is 'pilatus300': + if get_beamline().detector[0].name is "pilatus300k-1": md_current['detector_sequence_ID'] = caget('XF:11BMB-ES{Det:SAXS}:cam1:FileNumber_RBV') - elif get_beamline().detector[0].name is 'pilatus2M': + elif get_beamline().detector[0].name is "pilatus2m-1": md_current['detector_sequence_ID'] = caget('XF:11BMB-ES{Det:PIL2M}:cam1:FileNumber_RBV') md_current.update(get_beamline().get_md()) @@ -1327,10 +1327,10 @@ class Sample_Generic(CoordinateSystem): #caput('XF:11BMB-ES{}:cam1:AcquireTime'.format(pilatus_Epicsname), exposure_time) #caput('XF:11BMB-ES{}:cam1:AcquirePeriod'.format(pilatus_Epicsname), exposure_time+0.1) - if get_beamline().detector[0].name is 'pilatus300': + if get_beamline().detector[0].name is "pilatus300k-1": caput('XF:11BMB-ES{Det:SAXS}:cam1:AcquireTime', exposure_time) caput('XF:11BMB-ES{Det:SAXS}:cam1:AcquirePeriod', exposure_time+0.1) - elif get_beamline().detector[0].name is 'pilatus2M': + elif get_beamline().detector[0].name is "pilatus2m-1": caput('XF:11BMB-ES{Det:PIL2M}:cam1:AcquireTime', exposure_time) caput('XF:11BMB-ES{Det:PIL2M}:cam1:AcquirePeriod', exposure_time+0.1) @@ -1367,7 +1367,7 @@ class Sample_Generic(CoordinateSystem): if exposure_time is not None: #for detector in gs.DETS: for detector in get_beamline().detector: - if detector.name is 'pilatus2M' and exposure_time != caget('XF:11BMB-ES{Det:PIL2M}:cam1:AcquireTime'): + if detector.name is "pilatus2m-1" and exposure_time != caget('XF:11BMB-ES{Det:PIL2M}:cam1:AcquireTime'): detector.setExposureTime(exposure_time, verbosity=verbosity) #extra wait time when changing the exposure time. time.sleep(2) @@ -1398,10 +1398,10 @@ class Sample_Generic(CoordinateSystem): # Wait for detectors to be ready max_exposure_time = 0 for detector in get_beamline().detector: - if detector.name is 'pilatus300': + if detector.name is "pilatus300k-1": current_exposure_time = caget('XF:11BMB-ES{Det:SAXS}:cam1:AcquireTime') max_exposure_time = max(max_exposure_time, current_exposure_time) - elif detector.name is 'pilatus2M': + elif detector.name is "pilatus2m-1": current_exposure_time = caget('XF:11BMB-ES{Det:PIL2M}:cam1:AcquireTime') max_exposure_time = max(max_exposure_time, current_exposure_time) elif detector.name is 'PhotonicSciences_CMS': @@ -1420,10 +1420,10 @@ class Sample_Generic(CoordinateSystem): status = 1 for detector in get_beamline().detector: - if detector.name is 'pilatus300': + if detector.name is "pilatus300k-1": if caget('XF:11BMB-ES{Det:SAXS}:cam1:Acquire')==1: status *= 0 - elif detector.name is 'pilatus2M': + elif detector.name is "pilatus2m-1": if caget('XF:11BMB-ES{Det:PIL2M}:cam1:Acquire')==1: status *= 0 elif detector.name is 'PhotonicSciences_CMS': @@ -1454,7 +1454,7 @@ class Sample_Generic(CoordinateSystem): subdir = '' - if detector.name is 'pilatus300': + if detector.name is "pilatus300k-1": chars = caget('XF:11BMB-ES{Det:SAXS}:TIFF1:FullFileName_RBV') filename = ''.join(chr(char) for char in chars)[:-1] @@ -1489,7 +1489,7 @@ class Sample_Generic(CoordinateSystem): if verbosity>=3: print(' Data linked as: {}'.format(link_name)) - elif detector.name is 'pilatus2M': + elif detector.name is "pilatus2m-1": chars = caget('XF:11BMB-ES{Det:PIL2M}:TIFF1:FullFileName_RBV') filename = ''.join(chr(char) for char in chars)[:-1] @@ -1754,7 +1754,7 @@ class Sample_Generic(CoordinateSystem): # Wait for detectors to be ready max_exposure_time = 0 for detector in get_beamline().detector: - if detector.name is 'pilatus300' or 'pilatus2M': + if detector.name is "pilatus300k-1" or "pilatus2m-1": current_exposure_time = caget('XF:11BMB-ES{}:cam1:AcquireTime'.format(pilatus_Epicsname)) max_exposure_time = max(max_exposure_time, current_exposure_time) elif detector.name is 'PhotonicSciences_CMS': @@ -1780,7 +1780,7 @@ class Sample_Generic(CoordinateSystem): status = 1 for detector in get_beamline().detector: - if detector.name is 'pilatus300' or 'pilatus2M': + if detector.name is "pilatus300k-1" or "pilatus2m-1": print('status2.5 = ', status) if caget('XF:11BMB-ES{}:cam1:Acquire'.format(pilatus_Epicsname))==1: status = 0 diff --git a/startup/95-sample-custom.py b/startup/95-sample-custom.py index 155f30c..eb4eff0 100644 --- a/startup/95-sample-custom.py +++ b/startup/95-sample-custom.py @@ -973,7 +973,7 @@ def do(self, step=0, align_step=0, **md): # detselect(pilatus300) detselect(pilatus2M) for detector in get_beamline().detector: - if detector.name == "pilatus2M": + if detector.name == "pilatus2m-1": RE(detector.setExposureTime(self.md["exposure_time"])) else: detector.setExposureTime(self.md["exposure_time"]) diff --git a/startup/992-PTA-mobile.py b/startup/992-PTA-mobile.py index 1646fb6..65edaab 100644 --- a/startup/992-PTA-mobile.py +++ b/startup/992-PTA-mobile.py @@ -150,7 +150,7 @@ # The SmarAct module is broken. Need to change to a SPARE_S for armr # changed from spareM to spareS by RL at 2011/07/23 -armr = EpicsMotor("XF:11BMB-ES{Spare:L-Ax:S}Mtr", name="armr") +# armr = EpicsMotor("XF:11BMB-ES{Spare:L-Ax:S}Mtr", name="armr") ## stages for detectors diff --git a/startup/beamstop_config.cfg b/startup/beamstop_config.cfg index 1eae207..7045aeb 100644 --- a/startup/beamstop_config.cfg +++ b/startup/beamstop_config.cfg @@ -30,5 +30,13 @@ "bsphi": -64.001781, "timestamp": "2025-04-22 10:01:03" } + ], + "test": [ + { + "bsx": -17.900453, + "bsy": -10.098904, + "bsphi": 26.995626999999992, + "timestamp": "2025-07-28 12:55:37" + } ] } \ No newline at end of file diff --git a/startup/user_collection/user_LinkamThermal.py b/startup/user_collection/user_LinkamThermal.py index c3f570d..ca24da9 100644 --- a/startup/user_collection/user_LinkamThermal.py +++ b/startup/user_collection/user_LinkamThermal.py @@ -249,20 +249,20 @@ def series_measure( RE(detector.setExposurePeriod(exposure_period)) RE(detector.setExposureNumber(num_frames)) - # if detector.name is 'pilatus2M': + # if detector.name is "pilatus2m-1": # if exposure_time != caget('XF:11BMB-ES{Det:PIL2M}:cam1:AcquireTime'): # caput('XF:11BMB-ES{Det:PIL2M}:cam1:AcquireTime', exposure_time) # caput('XF:11BMB-ES{Det:PIL2M}:cam1:AcquirePeriod', exposure_period) # caput('XF:11BMB-ES{Det:PIL2M}:cam1:NumImages', num_frames) - # if detector.name is 'pilatus800': + # if detector.name is "pilatus800k-1": # if exposure_time != caget('XF:11BMB-ES{Det:PIL800K}:cam1:AcquireTime'): # caput('XF:11BMB-ES{Det:PIL800K}:cam1:AcquireTime', exposure_time) # caput('XF:11BMB-ES{Det:PIL800K}:cam1:AcquirePeriod', exposure_period) # caput('XF:11BMB-ES{Det:PIL800K}:cam1:NumImages', num_frames) - # if detector.name is 'pilatus300' : + # if detector.name is "pilatus300k-1" : # if exposure_time != caget('XF:11BMB-ES{Det:SAXS}:cam1:AcquireTime'): # caput('XF:11BMB-ES{Det:SAXS}:cam1:AcquireTime', exposure_time) # caput('XF:11BMB-ES{Det:SAXS}:cam1:AcquirePeriod', exposure_period) From 117e170b92bed94ceab5ea3ccbe74de1fe6dc066 Mon Sep 17 00:00:00 2001 From: CMS Operator Date: Mon, 28 Jul 2025 17:21:25 -0400 Subject: [PATCH 10/15] Removed local symlink gnerator --- startup/02-data_security.py | 3 +- startup/94-sample.py | 109 ++++++++++++++++++------------------ 2 files changed, 55 insertions(+), 57 deletions(-) diff --git a/startup/02-data_security.py b/startup/02-data_security.py index 4fb2bb0..e0d291f 100644 --- a/startup/02-data_security.py +++ b/startup/02-data_security.py @@ -8,5 +8,4 @@ RE.md.pop('experiment_group', None) RE.md.pop('experiment_cycle', None) -RE.md["experiment_alias_directory"] = "data_security_test" #NOTE TO SELF: Don't forget to delte this -RE.md['savename'] = "testname" \ No newline at end of file +RE.md["experiment_alias_directory"] = "data_security_test" #NOTE TO SELF: Don't forget to delte this \ No newline at end of file diff --git a/startup/94-sample.py b/startup/94-sample.py index dc70300..ce018ae 100644 --- a/startup/94-sample.py +++ b/startup/94-sample.py @@ -1900,7 +1900,7 @@ def expose(self, exposure_time=None, extra=None, handlefile=True, verbosity=3, p # Wait for detectors to be ready max_exposure_time = 0.1 for detector in get_beamline().detector: - print('here in expose:', detector.name) + # print('here in expose:', detector.name) if detector.name == "pilatus300k-1": current_exposure_time = detector.cam.acquire_time.get() max_exposure_time = max(max_exposure_time, current_exposure_time) @@ -2150,9 +2150,6 @@ def handle_file(self, detector, extra=None, verbosity=3, subdirs=True, linksave= return filename = detector.tiff.full_file_name.get() # RL, 20210831 - if not os.path.isfile(filename): - print("File does not exist") - return # Alternate method to get the last filename # filename = '{:s}/{:s}.tiff'.format( detector.tiff.file_path.get(), detector.tiff.file_name.get() ) @@ -2171,27 +2168,28 @@ def handle_file(self, detector, extra=None, verbosity=3, subdirs=True, linksave= # savename = self.get_savename(savename_extra=extra) savename = md["filename"] # link_name = '{}/{}{}_{:04d}_maxs.tiff'.format(RE.md['experiment_alias_directory'], subdir, savename, RE.md['scan_id']-1) - link_name = "{}/{}{}_{}.tiff".format(RE.md["experiment_alias_directory"], subdir, savename, detname) - - if os.path.isfile(link_name): - i = 1 - while os.path.isfile("{}.{:d}".format(link_name, i)): - i += 1 - os.rename(link_name, "{}.{:d}".format(link_name, i)) - os.symlink(filename, link_name) - - - #debug the losing data issue on pil2m. suggested by T. Caswell - # with open(link_name, 'rb') as fin: - # h = hashlib.md5(fin.read(1024)).hexdigest() - # with open(link_name + '.md5', 'w') as fout: - # fout.write(h) - - - if verbosity >= 3: - print(" Data linked as: {}".format(link_name)) - if not os.path.isfile(os.readlink(link_name)): #added by RL, 20231109 - raise ValueError('NO IMAGE OUTPUT.') + link_name = "{}/{}{}_000000_{}.tiff".format(RE.md["experiment_alias_directory"], subdir, savename, detname) + print(f" A symlink will be created at: {proposal_path()}experiments/{link_name}") + + # if os.path.isfile(link_name): + # i = 1 + # while os.path.isfile("{}.{:d}".format(link_name, i)): + # i += 1 + # os.rename(link_name, "{}.{:d}".format(link_name, i)) + # os.symlink(filename, link_name) + + + # #debug the losing data issue on pil2m. suggested by T. Caswell + # # with open(link_name, 'rb') as fin: + # # h = hashlib.md5(fin.read(1024)).hexdigest() + # # with open(link_name + '.md5', 'w') as fout: + # # fout.write(h) + + + # if verbosity >= 3: + # print(" Data linked as: {}".format(link_name)) + # if not os.path.isfile(os.readlink(link_name)): #added by RL, 20231109 + # raise ValueError('NO IMAGE OUTPUT.') def _old_handle_file(self, detector, extra=None, verbosity=3, subdirs=True, linksave=True, **md): subdir = "" @@ -3742,42 +3740,43 @@ def handle_fileseries(self, detector, num_frames=None, extra=None, verbosity=3, # savename = md['filename'][:-5] savename = self.get_savename(savename_extra=extra) - link_name = "{}/{}{}_{:06d}_{}.tiff".format( - RE.md["experiment_alias_directory"], - subdir, - savename, - RE.md["scan_id"] - 1, - detname, - ) - link_name_part1 = "{}/{}{}_{:06d}".format( - RE.md["experiment_alias_directory"], - subdir, - savename, - RE.md["scan_id"] - 1, - ) + # link_name = "{}/{}{}_{:06d}_{}.tiff".format( + # RE.md["experiment_alias_directory"], + # subdir, + # savename, + # RE.md["scan_id"] - 1, + # detname, + # ) + # link_name_part1 = "{}/{}{}_{:06d}".format( + # RE.md["experiment_alias_directory"], + # subdir, + # savename, + # RE.md["scan_id"] - 1, + # ) # link_name = '{}/{}{}_{:06d}_{}.tiff'.format(RE.md['experiment_alias_directory'], subdir, savename, RE.md['scan_id'], detname) # link_name_part1 = '{}/{}{}_{:06d}'.format(RE.md['experiment_alias_directory'], subdir, savename, RE.md['scan_id']) - if os.path.isfile(link_name): - i = 1 - while os.path.isfile("{}.{:d}".format(link_name, i)): - i += 1 - os.rename(link_name, "{}.{:d}".format(link_name, i)) - - for num_frame in range(num_frames): - filename_new = "{}_{:06d}.tiff".format(filename_part1, num_frame) - if os.path.isfile(filename_new) == False: - return print("File number {} does not exist.".format(num_frame)) - - link_name_new = "{}_{:06d}_{}.tiff".format(link_name_part1, num_frame, detname) - os.symlink(filename_new, link_name_new) - if verbosity >= 3: - if num_frame == 0 or num_frame == np.max(num_frames): - print(" Data {} linked as: {}".format(filename_new, link_name_new)) + # if os.path.isfile(link_name): + # i = 1 + # while os.path.isfile("{}.{:d}".format(link_name, i)): + # i += 1 + # os.rename(link_name, "{}.{:d}".format(link_name, i)) + + # for num_frame in range(num_frames): + # filename_new = "{}_{:06d}.tiff".format(filename_part1, num_frame) + # if os.path.isfile(filename_new) == False: + # return print("File number {} does not exist.".format(num_frame)) + + # link_name_new = "{}_{:06d}_{}.tiff".format(link_name_part1, num_frame, detname) + # os.symlink(filename_new, link_name_new) + # if verbosity >= 3: + # if num_frame == 0 or num_frame == np.max(num_frames): + # print(" Data {} linked as: {}".format(filename_new, link_name_new)) savename = self.get_savename(savename_extra=extra) # savename = md['filename'] # link_name = '{}/{}{}_{:04d}_maxs.tiff'.format(RE.md['experiment_alias_directory'], subdir, savename, RE.md['scan_id']-1) - link_name = "{}/{}{}_{}.tiff".format(RE.md["experiment_alias_directory"], subdir, savename, detname) + link_name = "{}/{}{}_000000_{}.tiff".format(RE.md["experiment_alias_directory"], subdir, savename, detname) + print(f" Symlinks will be created at: {proposal_path()}experiments/{link_name}") # Control methods ######################################## From 8badbe168ba7a79c961ca82c70b01cdc5a046f2f Mon Sep 17 00:00:00 2001 From: CMS Operator Date: Tue, 29 Jul 2025 17:06:08 -0400 Subject: [PATCH 11/15] Fixed use of file paths and names --- .gitignore | 5 +- startup/00-startup.py | 2 +- startup/02-data_security.py | 2 +- startup/81-beam.py | 100 ++++++++++++++++++++++++++++++++++++ startup/94-sample.py | 36 ++++++------- 5 files changed, 121 insertions(+), 24 deletions(-) diff --git a/.gitignore b/.gitignore index aea5894..be59cc8 100755 --- a/.gitignore +++ b/.gitignore @@ -143,4 +143,7 @@ doc/source/_generated_images/* *.visionenv # Audio Files -*.wav \ No newline at end of file +*.wav + +# Users scripts +users/ diff --git a/startup/00-startup.py b/startup/00-startup.py index 0543680..abb26d2 100644 --- a/startup/00-startup.py +++ b/startup/00-startup.py @@ -3,7 +3,7 @@ import nslsii import os -os.environ.pop('TILED_API_KEY') +os.environ.pop('TILED_API_KEY') # Make sure no user-defined API key is set from tiled.client import from_profile from databroker import Broker diff --git a/startup/02-data_security.py b/startup/02-data_security.py index e0d291f..1392b81 100644 --- a/startup/02-data_security.py +++ b/startup/02-data_security.py @@ -8,4 +8,4 @@ RE.md.pop('experiment_group', None) RE.md.pop('experiment_cycle', None) -RE.md["experiment_alias_directory"] = "data_security_test" #NOTE TO SELF: Don't forget to delte this \ No newline at end of file +RE.md["experiment_alias_directory"] = "data_security_test" #NOTE TO SELF: Don't forget to delete this \ No newline at end of file diff --git a/startup/81-beam.py b/startup/81-beam.py index 234ea5f..1955db0 100644 --- a/startup/81-beam.py +++ b/startup/81-beam.py @@ -3130,6 +3130,106 @@ def setMetadata(self, verbosity=3): "/n The folder ::: {} ::: has been made for users. /n".format(RE.md["experiment_alias_directory"]) ) + def setMetadata_new(self, verbosity=3): + """Guides the user through setting some of the required and recommended + meta-data fields.""" + + if verbosity >= 3: + print("This will guide you through adding some meta-data for the upcoming experiment.") + if verbosity >= 4: + print( + "You can accept default values (shown in square [] brackets) by pressing enter. You can leave a value blank (or put a space) to skip that entry." + ) + + # Set some values automatically + month = int(time.strftime("%m")) + if month <= 4: + cycle = 1 + elif month <= 8: + cycle = 2 + else: + cycle = 3 + RE.md["experiment_cycle"] = "{:s}_{:d}".format(time.strftime("%Y"), cycle) + + RE.md["calibration_energy_keV"] = float(round(self.beam.energy(verbosity=0), 3)) + RE.md["calibration_wavelength_A"] = float(round(self.beam.wavelength(verbosity=0), 5)) + + # TODO: + # RE.md['calibration_detector_distance_m'] = + # RE.md['calibration_detector_x0'] = + # RE.md['calibration_detector_y0'] = + + # Ask the user some questions + + questions = [ + ["experiment_proposal_number", "Proposal number"], + ["experiment_SAF_number", "SAF number"], + ["experiment_group", "User group (e.g. PI)"], + ["experiment_user", "The specific user/person running the experiment"], + ["experiment_project", "Project name/code"], + ["userpy_alias_directory", "Alias directory"], + ["experiment_alias_directory", "Alias directory"], + [ + "experiment_type", + "Type of experiments/measurements (SAXS, GIWAXS, etc.)", + ], + ] + + # TBD: + # Path where data will be stored? + + self._dialog_total_questions = len(questions) + self._dialog_question_number = 1 + + for key, text in questions: + try: + self._ask_question(key, text) + except KeyboardInterrupt: + return + + if verbosity >= 4: + print("You can also add/edit metadata directly using the RE.md object.") + + #RE.md["userpy_alias_directory"] = '/home/xf11bm/.ipython/profile_collection/users/2025-2/TKoga' + if os.path.exists(RE.md["userpy_alias_directory"]): + print("/n The folder has existed. Please change folder name if necessary./n") + else: + os.makedirs(RE.md["userpy_alias_directory"], exist_ok=True) + + #for double-checking folders + #/nsls2//data/cms/shared/config/bluesky/profile_collection/users/ + print('user.py will be saved in folder: {}'.format(RE.md["userpy_alias_directory"])) + print('data will be saved in folder: {}'.format(RE.md["experiment_alias_directory"])) + print('redo cms.setMeadata() if any folder is wrong! ') + + # if os.path.exists(RE.md["experiment_alias_directory"]): + # print("/n The folder has existed. Please change folder name if necessary./n") + # else: + # os.makedirs(RE.md["experiment_alias_directory"], exist_ok=True) + # os.makedirs(os.path.join(RE.md["experiment_alias_directory"], "waxs"), exist_ok=True) + # os.makedirs( + # os.path.join(RE.md["experiment_alias_directory"], "waxs/raw"), + # exist_ok=True, + # ) + # os.makedirs( + # os.path.join(RE.md["experiment_alias_directory"], "waxs/analysis"), + # exist_ok=True, + # ) + # os.makedirs(os.path.join(RE.md["experiment_alias_directory"], "saxs"), exist_ok=True) + # os.makedirs( + # os.path.join(RE.md["experiment_alias_directory"], "saxs/raw"), + # exist_ok=True, + # ) + # os.makedirs( + # os.path.join(RE.md["experiment_alias_directory"], "saxs/analysis"), + # exist_ok=True, + # ) + # os.makedirs(os.path.join(RE.md["experiment_alias_directory"], "data"), exist_ok=True) + # os.makedirs(os.path.join(RE.md['experiment_alias_directory'], 'saxs'), exist_ok=True) + # print( + # "/n The folder ::: {} ::: has been made for users. /n".format(RE.md["experiment_alias_directory"]) + # ) + def _ask_question(self, key, text, default=None): if default is None and key in RE.md: default = RE.md[key] diff --git a/startup/94-sample.py b/startup/94-sample.py index ce018ae..7c7ad79 100644 --- a/startup/94-sample.py +++ b/startup/94-sample.py @@ -2168,7 +2168,7 @@ def handle_file(self, detector, extra=None, verbosity=3, subdirs=True, linksave= # savename = self.get_savename(savename_extra=extra) savename = md["filename"] # link_name = '{}/{}{}_{:04d}_maxs.tiff'.format(RE.md['experiment_alias_directory'], subdir, savename, RE.md['scan_id']-1) - link_name = "{}/{}{}_000000_{}.tiff".format(RE.md["experiment_alias_directory"], subdir, savename, detname) + link_name = "{}/{}{}_000000_{}.tiff".format(RE.md["experiment_alias_directory"], subdir, savename, detname).replace('//','/') print(f" A symlink will be created at: {proposal_path()}experiments/{link_name}") # if os.path.isfile(link_name): @@ -2809,7 +2809,7 @@ def measureRock( md_current["sample_savename"] = savename md_current["measure_type"] = measure_type # md_current['filename'] = '{:s}_{:04d}.tiff'.format(savename, md_current['detector_sequence_ID']) - md_current["filename"] = "{:s}_{:04d}.tiff".format(savename, RE.md["scan_id"]) + md_current["filename"] = "{:s}_{:04d}".format(savename, RE.md["scan_id"]) md_current["beam_int_bim3"] = beam.bim3.flux(verbosity=0) md_current["beam_int_bim4"] = beam.bim4.flux(verbosity=0) md_current["beam_int_bim5"] = beam.bim5.flux(verbosity=0) @@ -2919,8 +2919,8 @@ def measure_single(self, exposure_time=None, extra=None, measure_type="measure", md_current.update(self.get_measurement_md()) md_current["sample_savename"] = savename md_current["measure_type"] = measure_type - # md_current['filename'] = '{:s}_{:04d}.tiff'.format(savename, md_current['detector_sequence_ID']) - # md_current['filename'] = '{:s}_{:04d}.tiff'.format(savename, RE.md['scan_id']) + # md_current['filename'] = '{:s}_{:04d}'.format(savename, md_current['detector_sequence_ID']) + # md_current['filename'] = '{:s}_{:04d}'.format(savename, RE.md['scan_id']) md_current["filename"] = "{:s}_{:06d}".format(savename, RE.md["scan_id"]) md_current.update(md) @@ -2968,8 +2968,8 @@ def _test_measure_single( md_current["measure_type"] = measure_type md_current.update(self.get_measurement_md()) - # md_current['filename'] = '{:s}_{:04d}.tiff'.format(savename, md_current['detector_sequence_ID']) - md_current["filename"] = "{:s}_{:04d}.tiff".format(savename, RE.md["scan_id"]) + # md_current['filename'] = '{:s}_{:04d}'.format(savename, md_current['detector_sequence_ID']) + md_current["filename"] = "{:s}_{:04d}".format(savename, RE.md["scan_id"]) md_current.update(md) self._test_expose(exposure_time, shutteronoff=shutteronoff, extra=extra, verbosity=verbosity, **md_current) self.md["measurement_ID"] += 1 @@ -3383,7 +3383,7 @@ def scan_measure( md_current["scan"] = "scan_measure" md_current.update(self.get_measurement_md()) md_current["measure_series_num_frames"] = num_frames - md_current["filename"] = "{:s}_{:04d}.tiff".format(savename, RE.md["scan_id"]) + md_current["filename"] = "{:s}_{:04d}".format(savename, RE.md["scan_id"]) md_current["measure_series_motor"] = motor.name md_current["measure_series_positions"] = [start, stop] md_current["exposure_time"] = exposure_time @@ -3488,16 +3488,16 @@ def series_measure( md_current["measure_type"] = measure_type md_current["series"] = "series_measure" md_current.update(self.get_measurement_md()) - # md_current['filename'] = '{:s}_{:04d}.tiff'.format(savename, md_current['detector_sequence_ID']) + # md_current['filename'] = '{:s}_{:04d}'.format(savename, md_current['detector_sequence_ID']) md_current["measure_series_num_frames"] = num_frames - md_current["filename"] = "{:s}_{:04d}.tiff".format(savename, RE.md["scan_id"]) - # md_current['filename'] = '{:s}_{:04d}.tiff'.format(savename, RE.md['scan_id']+1) + md_current["filename"] = "{:s}_{:04d}".format(savename, RE.md["scan_id"]) + # md_current['filename'] = '{:s}_{:04d}'.format(savename, RE.md['scan_id']+1) md_current["exposure_time"] = exposure_time md_current["exposure_period"] = exposure_period # md_current['measure_series_motor'] = motor.name # md_current['measure_series_positions'] = [start, stop] - # md_current['fileno'] = '{:s}_{:04d}.tiff'.format(savename, RE.md['scan_id']) + # md_current['fileno'] = '{:s}_{:04d}'.format(savename, RE.md['scan_id']) md_current.update(md) print(RE.md["scan_id"]) @@ -3722,8 +3722,8 @@ def handle_fileseries(self, detector, num_frames=None, extra=None, verbosity=3, print("WARNING: Can't do file handling for detector '{}'.".format(detector.name)) return - filename = detector.tiff.full_file_name.get() # RL, 20210831 - filename_part1 = "{:s}/{:s}".format(detector.tiff.file_path.get(), detector.tiff.file_name.get()) + # filename = detector.tiff.full_file_name.get() # RL, 20210831 + # filename_part1 = "{:s}/{:s}".format(detector.tiff.file_path.get(), detector.tiff.file_name.get()) # Alternate method to get the last filename # filename = '{:s}/{:s}.tiff'.format( detector.tiff.file_path.get(), detector.tiff.file_name.get() ) @@ -3740,6 +3740,7 @@ def handle_fileseries(self, detector, num_frames=None, extra=None, verbosity=3, # savename = md['filename'][:-5] savename = self.get_savename(savename_extra=extra) + # savename = RE.md["filename"] # link_name = "{}/{}{}_{:06d}_{}.tiff".format( # RE.md["experiment_alias_directory"], # subdir, @@ -3756,12 +3757,6 @@ def handle_fileseries(self, detector, num_frames=None, extra=None, verbosity=3, # link_name = '{}/{}{}_{:06d}_{}.tiff'.format(RE.md['experiment_alias_directory'], subdir, savename, RE.md['scan_id'], detname) # link_name_part1 = '{}/{}{}_{:06d}'.format(RE.md['experiment_alias_directory'], subdir, savename, RE.md['scan_id']) - # if os.path.isfile(link_name): - # i = 1 - # while os.path.isfile("{}.{:d}".format(link_name, i)): - # i += 1 - # os.rename(link_name, "{}.{:d}".format(link_name, i)) - # for num_frame in range(num_frames): # filename_new = "{}_{:06d}.tiff".format(filename_part1, num_frame) # if os.path.isfile(filename_new) == False: @@ -3772,10 +3767,9 @@ def handle_fileseries(self, detector, num_frames=None, extra=None, verbosity=3, # if verbosity >= 3: # if num_frame == 0 or num_frame == np.max(num_frames): # print(" Data {} linked as: {}".format(filename_new, link_name_new)) - savename = self.get_savename(savename_extra=extra) # savename = md['filename'] # link_name = '{}/{}{}_{:04d}_maxs.tiff'.format(RE.md['experiment_alias_directory'], subdir, savename, RE.md['scan_id']-1) - link_name = "{}/{}{}_000000_{}.tiff".format(RE.md["experiment_alias_directory"], subdir, savename, detname) + link_name = "{}/{}{}_{}_000000_{}.tiff".format(RE.md["experiment_alias_directory"], subdir, savename, RE.md["scan_id"] - 1, detname).replace('//', '/') print(f" Symlinks will be created at: {proposal_path()}experiments/{link_name}") # Control methods From ca46e0174bdfb3f9278d325bef2e879eaed0ae53 Mon Sep 17 00:00:00 2001 From: CMS Operator Date: Wed, 30 Jul 2025 14:21:49 -0400 Subject: [PATCH 12/15] FIX: configure writable path for specfiles --- startup/02-data_security.py | 1 + startup/86-live-spec.py | 2 +- startup/94-sample.py | 92 ++++++++++++++++++++----------------- 3 files changed, 51 insertions(+), 44 deletions(-) diff --git a/startup/02-data_security.py b/startup/02-data_security.py index 1392b81..9687124 100644 --- a/startup/02-data_security.py +++ b/startup/02-data_security.py @@ -7,5 +7,6 @@ RE.md.pop('experiment_project', None) RE.md.pop('experiment_group', None) RE.md.pop('experiment_cycle', None) +RE.md.pop('userpy_alias_directory', None) RE.md["experiment_alias_directory"] = "data_security_test" #NOTE TO SELF: Don't forget to delete this \ No newline at end of file diff --git a/startup/86-live-spec.py b/startup/86-live-spec.py index 86d09f3..d68c761 100644 --- a/startup/86-live-spec.py +++ b/startup/86-live-spec.py @@ -10,7 +10,7 @@ def spec_factory(name, doc): - directory = proposal_path() + "experiments/data/ScanFiles" + directory = "/nsls2/data/cms/shared/config/operations/specfiles/" file_prefix = "cms_scan_" + time.strftime("%Y_%m_%d") # # skip multiple motor scans diff --git a/startup/94-sample.py b/startup/94-sample.py index 7c7ad79..119d827 100644 --- a/startup/94-sample.py +++ b/startup/94-sample.py @@ -1931,55 +1931,55 @@ def expose(self, exposure_time=None, extra=None, handlefile=True, verbosity=3, p if verbosity >= 1: print("WARNING: Didn't recognize detector '{}'.".format(detector.name)) - if verbosity >= 2: - status = 0 - while (status == 0) and (time.time() - start_time) < (max_exposure_time + 20): - percentage = 100 * (time.time() - start_time) / max_exposure_time - print( - "Exposing {:6.2f} s ({:3.0f}%) \r".format((time.time() - start_time), percentage), - end="", - ) - - time.sleep(poling_period) - - status = 1 - for detector in get_beamline().detector: - if detector.cam.acquire.get() == 1: - status *= 0 + # if verbosity >= 2: + # status = 0 + # while (status == 0) and (time.time() - start_time) < (max_exposure_time + 20): + # percentage = 100 * (time.time() - start_time) / max_exposure_time + # print( + # "Exposing {:6.2f} s ({:3.0f}%) \r".format((time.time() - start_time), percentage), + # end="", + # ) + + # time.sleep(poling_period) + + # status = 1 + # for detector in get_beamline().detector: + # if detector.cam.acquire.get() == 1: + # status *= 0 # print('counting .... percentage = {}'.format(percentage)) else: time.sleep(max_exposure_time) - # special solution for 2022_1/TKoga2 - if verbosity >= 5: - print("verbosity = {}.".format(verbosity)) - pct_threshold = 90 - while percentage < pct_threshold: - print("sth is wrong .... percentage = {} < {}%".format(percentage, pct_threshold)) - start_time = time.time() - uids = RE(count(get_beamline().detector), **md) - # yield from (count(get_beamline().detector), **md) - - # get_beamline().beam.off() - # print('shutter is off') - - # Wait for detectors to be ready - max_exposure_time = 0.1 - for detector in get_beamline().detector: - if detector.name == "pilatus300k-1": - current_exposure_time = detector.cam.acquire_time.get() - max_exposure_time = max(max_exposure_time, current_exposure_time) - elif detector.name == "pilatus2m-1": - current_exposure_time = detector.cam.acquire_time.get() - max_exposure_time = max(max_exposure_time, current_exposure_time) - elif detector.name == "pilatus800k-1" or detector.name == "pilatus800k-2": - current_exposure_time = detector.cam.acquire_time.get() - max_exposure_time = max(max_exposure_time, current_exposure_time) - - percentage = 100 * (time.time() - start_time) / max_exposure_time - print("After re-exposing .... percentage = {} ".format(percentage)) + # # special solution for 2022_1/TKoga2 + # if verbosity >= 5: + # print("verbosity = {}.".format(verbosity)) + # pct_threshold = 90 + # while percentage < pct_threshold: + # print("sth is wrong .... percentage = {} < {}%".format(percentage, pct_threshold)) + # start_time = time.time() + # uids = RE(count(get_beamline().detector), **md) + # # yield from (count(get_beamline().detector), **md) + + # # get_beamline().beam.off() + # # print('shutter is off') + + # # Wait for detectors to be ready + # max_exposure_time = 0.1 + # for detector in get_beamline().detector: + # if detector.name == "pilatus300k-1": + # current_exposure_time = detector.cam.acquire_time.get() + # max_exposure_time = max(max_exposure_time, current_exposure_time) + # elif detector.name == "pilatus2m-1": + # current_exposure_time = detector.cam.acquire_time.get() + # max_exposure_time = max(max_exposure_time, current_exposure_time) + # elif detector.name == "pilatus800k-1" or detector.name == "pilatus800k-2": + # current_exposure_time = detector.cam.acquire_time.get() + # max_exposure_time = max(max_exposure_time, current_exposure_time) + + # percentage = 100 * (time.time() - start_time) / max_exposure_time + # print("After re-exposing .... percentage = {} ".format(percentage)) # if detector.name is "pilatus300k-1": # if caget('XF:11BMB-ES{Det:SAXS}:cam1:Acquire')==1: @@ -2144,6 +2144,10 @@ def handle_file(self, detector, extra=None, verbosity=3, subdirs=True, linksave= elif detector.name == "pilatus800k-1": subdir = "/waxs/raw/" detname = "waxs" + elif 'webcam' in detector.name: + subdir = "/camera/" + detname = detector.name + else: if verbosity >= 1: print("WARNING: Can't do file handling for detector '{}'.".format(detector.name)) @@ -2169,6 +2173,8 @@ def handle_file(self, detector, extra=None, verbosity=3, subdirs=True, linksave= savename = md["filename"] # link_name = '{}/{}{}_{:04d}_maxs.tiff'.format(RE.md['experiment_alias_directory'], subdir, savename, RE.md['scan_id']-1) link_name = "{}/{}{}_000000_{}.tiff".format(RE.md["experiment_alias_directory"], subdir, savename, detname).replace('//','/') + # if 'camera' in detector.name: + # link_name = "{}/{}{}_000000_{}.png".format(RE.md["experiment_alias_directory"], subdir, savename, detname).replace('//','/') print(f" A symlink will be created at: {proposal_path()}experiments/{link_name}") # if os.path.isfile(link_name): From 41bfa657e029a0b545bafc00a8f48d5e92cb062d Mon Sep 17 00:00:00 2001 From: CMS Operator Date: Wed, 30 Jul 2025 18:25:10 -0400 Subject: [PATCH 13/15] ENH: use databroker and fix legacy paths --- startup/.cms_config | 5 +++-- startup/00-startup.py | 2 +- startup/10-motors.py | 4 ++-- startup/91-fit_scan.py | 17 +++++++++-------- startup/94-sample.py | 2 +- 5 files changed, 16 insertions(+), 14 deletions(-) diff --git a/startup/.cms_config b/startup/.cms_config index 66d6966..53348dc 100644 --- a/startup/.cms_config +++ b/startup/.cms_config @@ -1312,5 +1312,6 @@ 1310,44.45,62.8,5.0,4.0,"[-96.3, -197.0, -122.5, -1.0]","[0.0, -101.80002499999999, -93.79981875, 89.2]","[0.0, -104.9, 0.0, 90.0]","[-97.5121875, -100.80002499999999, -93.79981875, 89.2]","[51.5, -1.87]","[-30.0, -2.37]",,,-17.900451,Thu Jul 24 16:37:48 2025 1311,44.45,62.8,5.0,4.0,"[-96.3, -197.0, -122.5, -1.0]","[0.0, -101.80002499999999, -93.79981875, 89.2]","[0.0, -104.9, 0.0, 90.0]","[-97.5121875, -100.80002499999999, -93.79981875, 89.2]","[51.5, -1.87]","[-30.0, -2.37]",,,-17.900453,Thu Jul 24 16:38:26 2025 1312,44.45,62.8,5.0,4.0,"[-96.3, -197.0, -122.5, -1.0]","[0.0, -101.80002499999999, -93.79981875, 89.2]","[0.0, -104.9, 0.0, 90.0]","[-97.5121875, -100.80002499999999, -93.79981875, 89.2]","[51.5, -1.87]","[-30.0, -2.37]",,,-17.900453,Thu Jul 24 16:40:16 2025 -1233,44.45,62.8,5.0,4.0,"[-96.3, -198.0, -122.5, -1.0]","[0.0, -101.80002499999999, -93.79981875, 89.2]","[0.0, -104.9, 0.0, 90.0]","[-97.5121875, -100.80002499999999, -93.79981875, 89.2]","[51.5, -1.87]","[-30.0, -2.37]",,-1.7,-17.900453,Mon Jul 28 11:38:49 2025 -1313,44.45,62.8,5.0,4.0,"[-96.3, -197.0, -122.5, -1.0]","[0.0, -101.80002499999999, -93.79981875, 89.2]","[0.0, -104.9, 0.0, 90.0]","[-97.5121875, -100.80002499999999, -93.79981875, 89.2]","[51.5, -1.87]","[-30.0, -2.37]",,,-17.900600999999998,Tue Jul 29 17:14:22 2025 +1313,44.45,62.8,5.0,4.0,"[-96.3, -198.0, -122.5, -1.0]","[0.0, -101.80002499999999, -93.79981875, 89.2]","[0.0, -104.9, 0.0, 90.0]","[-97.5121875, -100.80002499999999, -93.79981875, 89.2]","[51.5, -1.87]","[-30.0, -2.37]",,-1.7,-17.900453,Mon Jul 28 11:38:49 2025 +1314,44.45,62.8,5.0,4.0,"[-96.3, -197.0, -122.5, -1.0]","[0.0, -101.80002499999999, -93.79981875, 89.2]","[0.0, -104.9, 0.0, 90.0]","[-97.5121875, -100.80002499999999, -93.79981875, 89.2]","[51.5, -1.87]","[-30.0, -2.37]",,,-17.900600999999998,Tue Jul 29 17:14:22 2025 +1315,44.45,62.8,5.0,4.0,"[-96.3, -197.0, -122.5, -1.0]","[0.0, -101.80002499999999, -93.79981875, 89.2]","[0.0, -104.9, 0.0, 90.0]","[-97.5121875, -100.80002499999999, -93.79981875, 89.2]","[51.5, -1.87]","[-30.0, -2.37]",,-1.7,-18.400622,Wed Jul 30 15:37:23 2025 diff --git a/startup/00-startup.py b/startup/00-startup.py index abb26d2..e43b2d6 100644 --- a/startup/00-startup.py +++ b/startup/00-startup.py @@ -58,7 +58,7 @@ def insert(self, name, doc): print("Initializing Tiled reading client...\nMake sure you check for duo push.") tiled_reading_client = cat = from_profile("nsls2", username=None)["cms"]["raw"] -# db = Broker(tiled_reading_client) +db = Broker(tiled_reading_client) # Keep for backcompatibility with older code that uses databroker from pyOlog.ophyd_tools import * diff --git a/startup/10-motors.py b/startup/10-motors.py index cfeee2a..fcd1a00 100644 --- a/startup/10-motors.py +++ b/startup/10-motors.py @@ -8,8 +8,8 @@ # class Slits(Device): # top = Cpt(EpicsMotor, '-Ax:T}Mtr') # bottom = Cpt(EpicsMotor, '-Ax:B}Mtr') -#beamline_stage = "default" #for AB, please also change Smpl2-Y from 3... to -5 -beamline_stage = 'open_MAXS' +beamline_stage = "default" #for AB, please also change Smpl2-Y from 3... to -5 +# beamline_stage = 'open_MAXS' # beamline_stage = 'BigHuber' print('Beamline_stage = {}'.format(beamline_stage)) diff --git a/startup/91-fit_scan.py b/startup/91-fit_scan.py index 3eb7423..f7d59a8 100644 --- a/startup/91-fit_scan.py +++ b/startup/91-fit_scan.py @@ -951,7 +951,7 @@ def fit_scan( detectors = get_beamline().detector plot_y = get_beamline().PLOT_Y - # plot_y = pilatus2M.stats4.total + # plot_y = pilatus2m-1.stats4.total # print("plot_y is {}".format(plot_y)) else: @@ -1037,7 +1037,8 @@ def fit_scan( # md['exposure_time'] = exposure_time_last # if plot_y=='pilatus300_stats4_total' or plot_y=='pilatus300_stats3_total': - if plot_y == "pilatus2M_stats4_total" or plot_y == "pilatus2M_stats3_total": + if plot_y == "pilatus2m-1_stats4_total" or plot_y == "pilatus2m-1_stats3_total": + #TODO: if plot_y == detector.name + '_stats4_total': remove_last_Pilatus_series() # check save_flg and save the scan data thru databroker @@ -1162,7 +1163,7 @@ def fit_edge( bec.enable_table() # if plot_y=='pilatus300_stats4_total' or plot_y=='pilatus300_stats3_total': - if plot_y == "pilatus2M_stats4_total" or plot_y == "pilatus2M_stats3_total": + if plot_y == "pilatus2m-1_stats4_total" or plot_y == "pilatus2m-1_stats3_total": remove_last_Pilatus_series() x0_guess = np.average(livetable.xdata) @@ -1398,7 +1399,7 @@ def _fit_scan(): # RE(scan(list(detectors), motor, start, stop, num, per_step=per_step, md=md), subs ) # if plot_y=='pilatus300_stats4_total' or plot_y=='pilatus300_stats3_total': - if plot_y == "pilatus2M_stats4_total" or plot_y == "pilatus2M_stats3_total": + if plot_y == "pilatus2m-1_stats4_total" or plot_y == "pilatus2m-1_stats3_total": remove_last_Pilatus_series() if fit is None: @@ -1518,17 +1519,17 @@ def ps( if uid == "-1": uid = -1 if det == "default": - if db[uid].start['detectors'][0] == "pilatus2M" and suffix == "default": - intensity_field = "pilatus2M_stats4_total" + if db[uid].start['detectors'][0] == "pilatus2m-1" and suffix == "default": + intensity_field = "pilatus2m-1_stats4_total" # intensity_field = "elm_sum_all" - elif db[uid].start['detectors'][0] == "pilatus2M": + elif db[uid].start['detectors'][0] == "pilatus2m-1": intensity_field = "elm" + suffix elif suffix == "default": intensity_field = db[uid].start['detectors'][0] + "_stats1_total" else: intensity_field = db[uid].start['detectors'][0] + suffix else: - if det == "pilatus2M" and suffix == "default": + if det == "pilatus2m-1" and suffix == "default": intensity_field = "elm_sum_all" elif det == "elm": intensity_field = "elm" + suffix diff --git a/startup/94-sample.py b/startup/94-sample.py index 23e808a..cb30ec6 100644 --- a/startup/94-sample.py +++ b/startup/94-sample.py @@ -4048,7 +4048,7 @@ def transmission_data_output(self): # beam.absorber_transmission_list = [1, 0.041, 0.0017425, 0.00007301075, 0.00000287662355, 0.000000122831826, 0.00000000513437] scan_id = h.start["scan_id"] I_bim5 = 0 # beam intensity from bim5 - I0 = dtable.pilatus2M_stats4_total + I0 = dtable['pilatus2m-1_stats4_total'] filename = h.start["sample_name"] exposure_time = h.start["sample_exposure_time"] From a8612408e03b71bc725611c69ec641c886eb3877 Mon Sep 17 00:00:00 2001 From: CMS Operator Date: Fri, 15 Aug 2025 10:17:16 -0400 Subject: [PATCH 14/15] UPD: configuration changes --- startup/.cms_config | 11 +++++++++++ startup/10-motors.py | 4 ++-- startup/94-sample.py | 3 +++ startup/beamstop_config.cfg | 30 ++++++++++++++++++++++++++++++ 4 files changed, 46 insertions(+), 2 deletions(-) diff --git a/startup/.cms_config b/startup/.cms_config index cb7dd27..f3fe37b 100644 --- a/startup/.cms_config +++ b/startup/.cms_config @@ -1313,3 +1313,14 @@ 1311,44.45,62.8,5.0,4.0,"[-96.3, -197.0, -122.5, -1.0]","[0.0, -101.80002499999999, -93.79981875, 89.2]","[0.0, -104.9, 0.0, 90.0]","[-97.5121875, -100.80002499999999, -93.79981875, 89.2]","[51.5, -1.87]","[-30.0, -2.37]",,,-17.900453,Thu Jul 24 16:38:26 2025 1312,44.45,62.8,5.0,4.0,"[-96.3, -197.0, -122.5, -1.0]","[0.0, -101.80002499999999, -93.79981875, 89.2]","[0.0, -104.9, 0.0, 90.0]","[-97.5121875, -100.80002499999999, -93.79981875, 89.2]","[51.5, -1.87]","[-30.0, -2.37]",,,-17.900453,Thu Jul 24 16:40:16 2025 1313,44.45,62.8,5.0,4.0,"[-96.3, -197.0, -122.5, -1.0]","[0.0, -101.80002499999999, -93.79981875, 89.2]","[0.0, -104.9, 0.0, 90.0]","[-97.5121875, -100.80002499999999, -93.79981875, 89.2]","[51.5, -1.87]","[-30.0, -2.37]",,,-17.900600999999998,Tue Jul 29 17:14:22 2025 +1314,44.45,62.8,5.0,4.0,"[-96.3, -197.0, -122.5, -1.0]","[0.0, -101.80002499999999, -93.79981875, 89.2]","[0.0, -104.9, 0.0, 90.0]","[-97.5121875, -100.80002499999999, -93.79981875, 89.2]","[51.5, -1.87]","[-30.0, -2.37]",,,-16.300307999999998,Wed Jul 30 19:32:07 2025 +1315,44.45,62.8,5.0,4.0,"[-96.3, -197.0, -122.5, -1.0]","[0.0, -101.80002499999999, -93.79981875, 89.2]","[0.0, -104.9, 0.0, 90.0]","[-97.5121875, -100.80002499999999, -93.79981875, 89.2]","[51.5, -1.87]","[-30.0, -2.37]",,,-16.900647,Wed Jul 30 19:33:32 2025 +1316,44.45,62.8,5.0,4.0,"[-96.3, -197.0, -122.5, -1.0]","[0.0, -101.80002499999999, -93.79981875, 89.2]","[0.0, -104.9, 0.0, 90.0]","[-97.5121875, -100.80002499999999, -93.79981875, 89.2]","[51.5, -1.87]","[-30.0, -2.37]",,,-17.900453,Thu Jul 31 10:21:45 2025 +1317,44.45,62.8,5.0,4.0,"[-96.3, -197.0, -122.5, -1.0]","[0.0, -101.80002499999999, -93.79981875, 89.2]","[0.0, -104.9, 0.0, 90.0]","[-97.5121875, -100.80002499999999, -93.79981875, 89.2]","[51.5, -1.87]","[-30.0, -2.37]",,,-17.800403,Thu Jul 31 10:22:06 2025 +1318,44.45,62.8,5.0,4.0,"[-96.3, -197.0, -122.5, -1.0]","[0.0, -101.80002499999999, -93.79981875, 89.2]","[0.0, -104.9, 0.0, 90.0]","[-97.5121875, -100.80002499999999, -93.79981875, 89.2]","[51.5, -1.87]","[-30.0, -2.37]",,,-18.000816,Thu Jul 31 10:23:39 2025 +1319,44.45,62.8,5.0,4.0,"[-96.3, -197.0, -122.5, -1.0]","[0.0, -101.80002499999999, -93.79981875, 89.2]","[0.0, -104.9, 0.0, 90.0]","[-97.5121875, -100.80002499999999, -93.79981875, 89.2]","[51.5, -1.87]","[-30.0, -2.37]",,,-18.000816,Thu Jul 31 10:23:50 2025 +1320,44.45,62.8,5.0,4.0,"[-96.3, -197.0, -122.5, -1.0]","[0.0, -101.80002499999999, -93.79981875, 89.2]","[0.0, -104.9, 0.0, 90.0]","[-97.5121875, -100.80002499999999, -93.79981875, 89.2]","[51.5, -1.87]","[-30.0, -2.37]",,,-18.000816,Thu Jul 31 12:07:26 2025 +1321,44.45,62.8,5.0,4.0,"[-96.3, -197.0, -122.5, -1.0]","[0.0, -101.80002499999999, -93.79981875, 89.2]","[0.0, -104.9, 0.0, 90.0]","[-97.5121875, -100.80002499999999, -93.79981875, 89.2]","[51.5, -1.87]","[-30.0, -2.37]",,,-17.900453,Thu Jul 31 12:07:39 2025 +1322,44.45,62.8,5.0,4.0,"[-96.3, -197.0, -122.5, -1.0]","[0.0, -101.80002499999999, -93.79981875, 89.2]","[0.0, -104.9, 0.0, 90.0]","[-97.5121875, -100.80002499999999, -93.79981875, 89.2]","[51.5, -1.87]","[-30.0, -2.37]",,,-18.200816,Thu Jul 31 15:23:43 2025 +1323,44.45,62.8,5.0,4.0,"[-96.3, -197.0, -122.5, -1.0]","[0.0, -101.80002499999999, -93.79981875, 89.2]","[0.0, -104.9, 0.0, 90.0]","[-97.5121875, -100.80002499999999, -93.79981875, 89.2]","[51.5, -1.87]","[-30.0, -2.37]",,,-18.000816,Thu Jul 31 15:33:26 2025 +1324,44.45,62.8,5.0,4.0,"[-96.3, -197.0, -122.5, -1.0]","[0.0, -101.80002499999999, -93.79981875, 89.2]","[0.0, -104.9, 0.0, 90.0]","[-97.5121875, -100.80002499999999, -93.79981875, 89.2]","[51.5, -1.87]","[-30.0, -2.37]",,,-16.800351,Mon Aug 4 17:20:11 2025 diff --git a/startup/10-motors.py b/startup/10-motors.py index cfeee2a..fcd1a00 100644 --- a/startup/10-motors.py +++ b/startup/10-motors.py @@ -8,8 +8,8 @@ # class Slits(Device): # top = Cpt(EpicsMotor, '-Ax:T}Mtr') # bottom = Cpt(EpicsMotor, '-Ax:B}Mtr') -#beamline_stage = "default" #for AB, please also change Smpl2-Y from 3... to -5 -beamline_stage = 'open_MAXS' +beamline_stage = "default" #for AB, please also change Smpl2-Y from 3... to -5 +# beamline_stage = 'open_MAXS' # beamline_stage = 'BigHuber' print('Beamline_stage = {}'.format(beamline_stage)) diff --git a/startup/94-sample.py b/startup/94-sample.py index b07fa09..4a8c938 100644 --- a/startup/94-sample.py +++ b/startup/94-sample.py @@ -1887,6 +1887,9 @@ def expose(self, exposure_time=None, extra=None, handlefile=True, verbosity=3, p md["beam_int_bim3"] = beam.bim3.flux(verbosity=0) md["beam_int_bim4"] = beam.bim4.flux(verbosity=0) md["beam_int_bim5"] = beam.bim5.flux(verbosity=0) + + if 'temperature_Linkam' in self.naming_scheme: + md["temperature_Linkam"] = LThermal.temperature() # md['trigger_time'] = self.clock() # md.update(md_current) diff --git a/startup/beamstop_config.cfg b/startup/beamstop_config.cfg index 39ac60c..14b199b 100755 --- a/startup/beamstop_config.cfg +++ b/startup/beamstop_config.cfg @@ -35,6 +35,30 @@ "bsy": -12.799769, "bsphi": -61.001744, "timestamp": "2025-06-26 17:04:33" + }, + { + "bsx": -18.000815999999997, + "bsy": -12.599539, + "bsphi": -61.003749, + "timestamp": "2025-07-31 10:23:39" + }, + { + "bsx": -18.000815999999997, + "bsy": -12.599539, + "bsphi": -61.003749, + "timestamp": "2025-07-31 10:23:50" + }, + { + "bsx": -18.200816, + "bsy": -12.599538999999998, + "bsphi": -61.00361100000001, + "timestamp": "2025-07-31 12:07:26" + }, + { + "bsx": -18.000816, + "bsy": -12.399296, + "bsphi": -61.005459, + "timestamp": "2025-07-31 15:33:26" } ], "round_17": [ @@ -117,6 +141,12 @@ "bsy": 17.001509, "bsphi": -181.005665, "timestamp": "2025-06-17 09:46:10" + }, + { + "bsx": -16.900647, + "bsy": 17.001970999999998, + "bsphi": -181.008503, + "timestamp": "2025-07-30 19:33:32" } ], "round_3m": [ From 442dd657e2786d31e65e313414f365e5c12678e5 Mon Sep 17 00:00:00 2001 From: jennmald Date: Tue, 30 Sep 2025 10:48:48 -0400 Subject: [PATCH 15/15] create xspress3 classes --- startup/26-xspress3.py | 515 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 515 insertions(+) create mode 100644 startup/26-xspress3.py diff --git a/startup/26-xspress3.py b/startup/26-xspress3.py new file mode 100644 index 0000000..82b16be --- /dev/null +++ b/startup/26-xspress3.py @@ -0,0 +1,515 @@ +from ophyd.areadetector import (AreaDetector, PixiradDetectorCam, ImagePlugin, + TIFFPlugin, StatsPlugin, HDF5Plugin, + ProcessPlugin, ROIPlugin, TransformPlugin, + OverlayPlugin, DetectorBase, ADBase) +from ophyd.areadetector.plugins import PluginBase +from ophyd.areadetector.cam import AreaDetectorCam +from ophyd.device import BlueskyInterface +from ophyd.areadetector.trigger_mixins import SingleTrigger +from ophyd.areadetector.filestore_mixins import (FileStoreIterativeWrite, + FileStoreHDF5IterativeWrite, + FileStoreTIFFSquashing, + FileStoreTIFF) +from ophyd import Signal, EpicsSignal, EpicsSignalRO +from ophyd.status import SubscriptionStatus +from ophyd.sim import NullStatus # TODO: remove after complete/collect are defined +from ophyd import Component as Cpt +from ophyd import DynamicDeviceComponent as DDC +from ophyd.status import SubscriptionStatus, DeviceStatus + +from pathlib import PurePath +from nslsii.detectors.xspress3 import (XspressTrigger, Xspress3Detector, Xspress3DetectorSettings, make_rois, + Xspress3Channel, Xspress3FileStore, Xspress3ROI, logger) + +# from isstools.trajectory.trajectory import trajectory_manager + +import bluesky.plans as bp +import bluesky.plan_stubs as bps + +import numpy as np +import itertools +import time as ttime +from collections import deque, OrderedDict +import warnings +from databroker.assets.handlers import XS3_XRF_DATA_KEY as XRF_DATA_KEY + + +class Xspress3FileStoreFlyable(Xspress3FileStore): + def warmup(self): + """ + A convenience method for 'priming' the plugin. + The plugin has to 'see' one acquisition before it is ready to capture. + This sets the array size, etc. + NOTE : this comes from: + https://github.com/NSLS-II/ophyd/blob/master/ophyd/areadetector/plugins.py + We had to replace "cam" with "settings" here. + Also modified the stage sigs. + """ + print("warming up the hdf5 plugin...") + self.enable.set(1).wait() + sigs = OrderedDict([(self.parent.settings.array_callbacks, 1), + (self.parent.settings.trigger_mode, 'Internal'), + # just in case the acquisition time is set very long... + (self.parent.settings.acquire_time, 1), + # (self.capture, 1), + (self.parent.settings.acquire, 1)]) + + original_vals = {sig: sig.get() for sig in sigs} + + # Remove the hdf5.capture item here to avoid an error as it should reset back to 0 itself + # del original_vals[self.capture] + + for sig, val in sigs.items(): + ttime.sleep(0.1) # abundance of caution + sig.set(val).wait() + + ttime.sleep(2) # wait for acquisition + + for sig, val in reversed(list(original_vals.items())): + ttime.sleep(0.1) + sig.set(val).wait() + print("done") + +class Xspress3XDetector(DetectorBase): + settings = Cpt(Xspress3DetectorSettings, 'det1:') + + external_trig = Cpt(Signal, value=False, + doc='Use external triggering') + total_points = Cpt(Signal, value=-1, + doc='The total number of points to acquire overall') + spectra_per_point = Cpt(Signal, value=1, + doc='Number of spectra per point') + make_directories = Cpt(Signal, value=False, + doc='Make directories on the DAQ side') + rewindable = Cpt(Signal, value=False, + doc='Xspress3 cannot safely be rewound in bluesky') + + # channel1 = Cpt(Xspress3Channel, 'C1_', channel_num=1) + # XF:11BM-ES{Xsp:1} + + data_key = XRF_DATA_KEY + + def __init__(self, prefix, *, read_attrs=None, configuration_attrs=None, + name=None, parent=None, + # to remove? + file_path='', ioc_file_path='', default_channels=None, + channel_prefix=None, + roi_sums=False, + # to remove? + **kwargs): + + if read_attrs is None: + read_attrs = ['channel1', ] + + if configuration_attrs is None: + configuration_attrs = ['channel1.rois', 'settings'] + + super().__init__(prefix, read_attrs=read_attrs, + configuration_attrs=configuration_attrs, + name=name, parent=parent, **kwargs) + + # get all sub-device instances + sub_devices = {attr: getattr(self, attr) + for attr in self._sub_devices} + + # filter those sub-devices, just giving channels + channels = {dev.channel_num: dev + for attr, dev in sub_devices.items() + if isinstance(dev, Xspress3Channel) + } + + # make an ordered dictionary with the channels in order + self._channels = OrderedDict(sorted(channels.items())) + + @property + def channels(self): + return self._channels.copy() + + @property + def all_rois(self): + for ch_num, channel in self._channels.items(): + for roi in channel.all_rois: + yield roi + + @property + def enabled_rois(self): + for roi in self.all_rois: + if roi.enable.get(): + yield roi + + def read_hdf5(self, fn, *, rois=None, max_retries=2): + '''Read ROI data from an HDF5 file using the current ROI configuration + + Parameters + ---------- + fn : str + HDF5 filename to load + rois : sequence of Xspress3ROI instances, optional + + ''' + if rois is None: + rois = self.enabled_rois + + num_points = self.settings.num_images.get() + if isinstance(fn, h5py.File): + hdf = fn + else: + hdf = h5py.File(fn, 'r') + + RoiTuple = Xspress3ROI.get_device_tuple() + + handler = Xspress3HDF5Handler(hdf, key=self.data_key) + for roi in self.enabled_rois: + roi_data = handler.get_roi(chan=roi.channel_num, + bin_low=roi.bin_low.get(), + bin_high=roi.bin_high.get(), + max_points=num_points) + + roi_info = RoiTuple(bin_low=roi.bin_low.get(), + bin_high=roi.bin_high.get(), + ev_low=roi.ev_low.get(), + ev_high=roi.ev_high.get(), + value=roi_data, + value_sum=None, + enable=None) + + yield roi.name, roi_info + + +class Xspress3XChannel(ADBase): + roi_name_format = 'Det{self.channel_num}_{roi_name}' + roi_sum_name_format = 'Det{self.channel_num}_{roi_name}_sum' + + rois = DDC(make_rois(range(1, 4))) + vis_enabled = Cpt(EpicsSignal, 'EnableCallbacks') + # extra_rois_enabled = Cpt(EpicsSignal, 'PluginControlValExtraROI') + + def __init__(self, prefix, *, channel_num=None, **kwargs): + self.channel_num = int(channel_num) + + super().__init__(prefix, **kwargs) + + @property + def all_rois(self): + for roi in range(1, self.rois.num_rois.get() + 1): + yield getattr(self.rois, 'roi{:02d}'.format(roi)) + + def set_roi(self, index, ev_low, ev_high, *, name=None): + '''Set specified ROI to (ev_low, ev_high) + + Parameters + ---------- + index : int or Xspress3ROI + The roi index or instance to set + ev_low : int + low eV setting + ev_high : int + high eV setting + name : str, optional + The unformatted ROI name to set. Each channel specifies its own + `roi_name_format` and `roi_sum_name_format` in which the name + parameter will get expanded. + ''' + if isinstance(index, Xspress3ROI): + roi = index + else: + if index <= 0: + raise ValueError('ROI index starts from 1') + roi = list(self.all_rois)[index - 1] + + roi.configure(ev_low, ev_high) + if name is not None: + roi_name = self.roi_name_format.format(self=self, roi_name=name) + roi.name = roi_name + roi.value.name = roi_name + roi.value_sum.name = self.roi_sum_name_format.format(self=self, + roi_name=name) + + def clear_all_rois(self): + '''Clear all ROIs''' + for roi in self.all_rois: + roi.clear() + +xsx = QASXspress3XDetector('XF:11BM-ES{Xsp:1}:', name='xspress3-1') + +class QASXspress3XDetector(XspressTrigger, Xspress3XDetector): + roi_data = Cpt(PluginBase, 'ROIDATA:') + channel1 = Cpt(Xspress3XChannel, 'C1_', channel_num=1, read_attrs=['rois']) + # create_dir = Cpt(EpicsSignal, 'HDF5:FileCreateDir') + + mca1_sum = Cpt(EpicsSignal, 'ARRSUM1:ArrayData') + + mca1 = Cpt(EpicsSignal, 'ARR1:ArrayData') + + cnt_time = Cpt(EpicsSignal, 'C1_SCA0:Value_RBV') + + # channel6 = Cpt(Xspress3Channel, 'C6_', channel_num=6) + + #TODO change folder to xspress3 + # hdf5 = Cpt(Xspress3FileStoreFlyable, 'HDF1:', + # read_path_template='/nsls2/data/qas-new/legacy/raw/x3x/%Y/%m/%d/', + # root='/nsls2/data/qas-new/legacy/raw/', + # write_path_template='/nsls2/data/qas-new/legacy/raw/x3x/%Y/%m/%d/', + # ) + hdf5 = Cpt(Xspress3FileStoreFlyable, 'HDF1:',write_path_template='',) + + def __init__(self, prefix, *, configuration_attrs=None, read_attrs=None, + **kwargs): + if configuration_attrs is None: + configuration_attrs = ['external_trig', 'total_points', + 'spectra_per_point', 'settings', + 'rewindable'] + if read_attrs is None: + read_attrs = ['channel1', 'hdf5', 'settings.acquire_time'] + super().__init__(prefix, configuration_attrs=configuration_attrs, + read_attrs=read_attrs, **kwargs) + self.set_channels_for_hdf5() + + self._asset_docs_cache = deque() + self._datum_counter = None + + self.channel1.rois.roi01.configuration_attrs.append('bin_low') + + # Step-scan interface methods. + def stage(self, *args, **kwargs): + if self.spectra_per_point.get() != 1: + raise NotImplementedError( + "multi spectra per point not supported yet") + self.hdf5.write_path_template = assets_path() + f'{xsx.name}/%Y/%m/%d/' + self.hdf5.read_path_template = assets_path() + f'{xsx.name}/%Y/%m/%d/' + self.hdf5.reg_root = assets_path() + f'{xsx.name}' + ret = super().stage(*args, **kwargs) + self._datum_counter = itertools.count() + return ret + + def trigger(self): + + self._status = DeviceStatus(self) + self.settings.erase.put(1) + self._acquisition_signal.put(1, wait=False) + trigger_time = ttime.time() + + for sn in self.read_attrs: + if sn.startswith('channel') and '.' not in sn: + ch = getattr(self, sn) + self.generate_datum(ch.name, trigger_time) + + self._abs_trigger_count += 1 + return self._status + + def unstage(self): + self.settings.trigger_mode.put(1) # 'Software' + super().unstage() + self._datum_counter = None + + def stop(self): + ret = super().stop() + self.hdf5.stop() + return ret + + def collect(self): + # TODO: try to separate it from the xspress3 class + collected_frames = self.settings.array_counter.get() + + # This is a hack around the issue with .NORD (number of elements to # + # read) that does not match .NELM (number of elements to that the array + # will hold) + dpb_sec_nelm_count = int(dpb_sec_nelm.get()) + dpb_nsec_nelm_count = int(dpb_nsec_nelm.get()) + dpb_sec_values = np.array(dpb_sec.get(count=dpb_sec_nelm_count), + dtype='float128')[:collected_frames * 2: 2] + dpb_nsec_values = np.array(dpb_nsec.get(count=dpb_nsec_nelm_count), + dtype='float128')[:collected_frames * 2: 2] + + di_timestamps = dpb_sec_values + dpb_nsec_values * 1e-9 + + len_di_timestamps = len(di_timestamps) + len_datum_ids = len(self._datum_ids) + + if len_di_timestamps != len_datum_ids: + warnings.warn(f'The length of "di_timestamps" ({len_di_timestamps}) ' + f'does not match the length of "self._datum_ids" ({len_datum_ids})') + + num_frames = min(len_di_timestamps, len_datum_ids) + num_frames = len_datum_ids + for frame_num in range(num_frames): + datum_id = self._datum_ids[frame_num] + # ts = di_timestamps[frame_num] + ts = di_timestamps + + data = {self.name: datum_id} + # TODO: fix the lost precision as pymongo complained about np.float128. + ts = float(ts) + + yield {'data': data, + 'timestamps': {key: ts for key in data}, + 'time': ts, # TODO: use the proper timestamps from the mono start and stop times + 'filled': {key: False for key in data}} + + def set_channels_for_hdf5(self, channels=(1)): + """ + Configure which channels' data should be saved in the resulted hdf5 file. + Parameters + ---------- + channels: tuple, optional + the channels to save the data for + """ + # The number of channel + for n in channels: + getattr(self, f'channel{n}').rois.read_attrs = ['roi{:02}'.format(j) for j in [1]] + self.hdf5.num_extra_dims.put(0) + self.settings.num_channels.put(8) + + +def initialize_Xspress3X(xsx, hdf5_warmup=True): + # TODO: do not put on startup or do it conditionally, if on beamline. + xsx.channel1.vis_enabled.put(1) + xsx.total_points.put(1) + + # This is necessary for when the ioc restarts + # we have to trigger one image for the hdf5 plugin to work correctly + # else, we get file writing errors + if hdf5_warmup: + xsx.hdf5.warmup() + + # Hints: + for n in [1, 2]: # TODO: 8? + getattr(xsx, f'channel{n}').rois.roi01.value.kind = 'hinted' + + xsx.settings.configuration_attrs = ['acquire_period', + 'acquire_time', + 'gain', + 'image_mode', + 'manufacturer', + 'model', + 'num_exposures', + 'num_images', + 'temperature', + 'temperature_actual', + 'trigger_mode', + 'config_path', + 'config_save_path', + 'invert_f0', + 'invert_veto', + 'xsp_name', + 'num_channels', + 'num_frames_config', + 'run_flags', + 'trigger_signal'] + + for n, d in xsx.channels.items(): + roi_names = ['roi{:02}'.format(j) for j in [1]] + d.rois.read_attrs = roi_names + d.rois.configuration_attrs = roi_names + for roi_n in roi_names: + getattr(d.rois, roi_n).value_sum.kind = 'omitted' + + +initialize_Xspress3X(xsx) + + +def xsx_count(acq_time: float = 1, num_frames: int = 1): + yield from bps.mv(xsx.settings.erase, 0) + yield from bp.count([xsx], acq_time) + + +class QASXspress3XDetectorStream(QASXspress3XDetector): + + def stage(self, acq_rate, traj_time, *args, **kwargs): + self.hdf5.file_write_mode.put(2) # put it to Stream |||| IS ALREADY STREAMING + self.external_trig.put(True) + self.set_expected_number_of_points(acq_rate, traj_time) + self.spectra_per_point.put(1) + self.settings.trigger_mode.put(3) # put the trigger mode to TTL in + + super().stage(*args, **kwargs) + # note, hdf5 is already capturing at this point + self.settings.acquire.put(1) # start recording data + + def set_expected_number_of_points(self, acq_rate, traj_time): + self._num_points = int(acq_rate * (traj_time + 1)) + self.total_points.put(self._num_points) + + def describe_collect(self): + return_dict = {self.name: + {f'{self.name}': {'source': 'XS', + 'dtype': 'array', + 'shape': [self.settings.num_images.get(), + self.hdf5.array_size.height.get(), + self.hdf5.array_size.width.get()], + 'filename': f'{self.hdf5.full_file_name.get()}', + 'external': 'FILESTORE:'}}} + return return_dict + + def collect(self): + num_frames = len(self._datum_ids) + + # break num_frames up and yield in sections? + + for frame_num in range(num_frames): + datum_id = self._datum_ids[frame_num] + data = {self.name: datum_id} + + ts = ttime.time() + + yield {'data': data, + 'timestamps': {key: ts for key in data}, + 'time': ts, # TODO: use the proper timestamps from the mono start and stop times + 'filled': {key: False for key in data}} + # print(f"-------------------{ts}-------------------------------------") + + def collect_asset_docs(self): + items = list(self._asset_docs_cache) + self._asset_docs_cache.clear() + for item in items: + yield item + +xsx_stream = QASXspress3XDetectorStream('XF:11BM-ES{Xsp:1}:', name='xsx_stream') +initialize_Xspress3X(xsx_stream, hdf5_warmup=True) + + +from itertools import product +import pandas as pd +from databroker.assets.handlers import HandlerBase, Xspress3HDF5Handler + + +class QASXspress3XHDF5Handler(Xspress3HDF5Handler): + + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + self._roi_data = None + self._num_channels = None + + def _get_dataset( + self): # readpout of the following stuff should be done only once, this is why I redefined _get_dataset method - Denis Leshchev Feb 9, 2021 + # dealing with parent + super()._get_dataset() + + # finding number of channels + if self._num_channels is not None: + return + print('determining number of channels') + shape = self.dataset.shape + if len(shape) != 3: + raise RuntimeError(f'The ndim of the dataset is not 3, but {len(shape)}') + self._num_channels = shape[1] + + if self._roi_data is not None: + return + print('reading ROI data') + self.chanrois = [f'CHAN{c}ROI{r}' for c, r in product([1], [1])] + _data_columns = [self._file['/entry/instrument/detector/NDAttributes'][chanroi][()] for chanroi in + self.chanrois] + data_columns = np.vstack(_data_columns).T + self._roi_data = pd.DataFrame(data_columns, columns=self.chanrois) + + def __call__(self, *args, frame=None, **kwargs): + self._get_dataset() + return_dict = {f'ch_{i + 1}': self._dataset[frame, i, :] for i in range(self._num_channels)} + return_dict_rois = {chanroi: self._roi_data[chanroi][frame] for chanroi in self.chanrois} + return {**return_dict, **return_dict_rois} + + +# heavy-weight file handler +db.reg.register_handler(QASXspress3XHDF5Handler.HANDLER_NAME, + QASXspress3XHDF5Handler, overwrite=True)