Update 2022 c2

azure-pipelines.yml

@@ -7,6 +7,6 @@ resources:
       endpoint: github
 
 jobs:
-  - template: collection-2021-1.0.yml@templates
+  - template: 2022-2.3-py39.yml@templates
     parameters:
       beamline_acronym: LIX

startup/00-base.py

@@ -60,7 +60,7 @@ def __call__(self, *args, **kwargs):
 
 RE = CustomRunEngine()
 
-nslsii.configure_base(get_ipython().user_ns, 'lix', bec=True, pbar=False)
+nslsii.configure_base(get_ipython().user_ns, 'lix', bec=True, pbar=False, publish_documents_with_kafka=True)
 
 def reload_macros(file='~/.ipython/profile_collection/startup/99-macros.py'):
     ipy = get_ipython()

startup/02-vars.py

@@ -1,2 +1,22 @@
-current_cycle = '2022-1'
-pilatus_data_dir = "/exp_path/hdf"
+from enum import Enum
+
+class data_file_path(Enum):
+    old_gpfs = '/GPFS/xf16id/exp_path'
+    lustre_legacy = '/nsls2/data/lix/legacy'
+    lustre_asset = '/nsls2/data/lix/asset'
+    lustre_proposals = '/nsls2/data/lix/proposals'
+    gpfs = '/nsls2/xf16id1/data'
+    gpfs_experiments = '/nsls2/xf16id1/experiments'
+    ramdisk = '/exp_path'
+
+current_cycle = '2022-2'
+pilatus_data_dir = data_file_path.lustre_legacy.value
+#pilatus_data_dir = f"{data_file_path.ramdisk.value}/hdf"
+data_destination = data_file_path.lustre_legacy.value  # this is where all IOC data files should eventually go
+#proc_destination = data_file_path.lustre_proposals.value
+proc_destination = data_file_path.gpfs_experiments.value
+procdir_prefix = "pass-"  
+
+bl_comm_proposal = "310032"
+
+

startup/03-security.py

@@ -1,16 +1,39 @@
-import os,stat,time,shutil
+import os
+import re
+import shutil
+import stat
+import subprocess
+import time
+
 from IPython import get_ipython
+from lixtools.atsas import run as run_cmd
 
 username = None
 proposal_id = None
 run_id = None
 data_path = ""
-collection_lock_file = "/nsls2/xf16id1/.lock"
-okay_to_move_file = "/nsls2/xf16id1/.okay_to_move"
+collection_lock_file = f"{data_file_path.lustre_legacy.value}/.lock"
 login_time = -1
 
-def login(uname = None, pID = None, rID = None, debug=True, test_only=False,
-          root_path='/nsls2/xf16id1', replace_froot=pilatus_data_dir, share_with=[]):
+def check_access(fn):
+    if not os.path.exists(fn):
+        raise Exception(f"{fn} does not exist ...")
+    if os.access(fn, os.W_OK):
+        print(f"write access to {fn} verified ...")
+        return
+
+    # this below may not be necessary
+    out = run_cmd(["getfacl", "-cn", fn])
+    wgrps = [int(t[:-4].lstrip("group:")) for t in re.findall("groups:[0-9]*:rw.", out)]
+    ugrps = os.getgroups()
+    if len(set(wgrps) & set(ugrps))==0:
+        print("groups with write permission: ", wgrps)
+        print("user group membership: ", ugrps)
+        raise Exception(f"the current user does not have write access to {fn}")
+    else:
+        print(f"write access to {fn} verified ...")
+
+def login(uname = None, pID = None, rID = None, debug=True, test_only=False):
     """ Ask the user for his credentials and proposal information for the data collection
         create_proc_dir: if True, create the directory where h5 files will be saved
         share_with: list of e-mails to share the proc_path with
@@ -20,7 +43,6 @@ def login(uname = None, pID = None, rID = None, debug=True, test_only=False,
     global proposal_id
     global run_id
     global data_path 
-    global o_data_path 
     global proc_path
     global login_time
 
@@ -35,8 +57,11 @@ def login(uname = None, pID = None, rID = None, debug=True, test_only=False,
         correct_info = True
     elif test_only:
         username = "lix"
-        proposal_id = "test"
-        run_id = "test"
+        proposal_id = bl_comm_proposal
+        if rID:
+            run_id = rID
+        else:
+            run_id = "test"
         correct_info = True
 
     while not correct_info:
@@ -51,28 +76,16 @@ def login(uname = None, pID = None, rID = None, debug=True, test_only=False,
     RE.md['proposal_id'] = proposal_id
     RE.md['run_id'] = run_id
     login_time = time.time()
-
-    if test_only:
-        path = f"{root_path}/data/"
-    else:
-        path = f"{root_path}/data/{current_cycle}/"
-    rpath = str(proposal_id)+"/"+str(run_id)+"/"
-    data_path = path + rpath
-    makedirs(data_path, mode=0o0777)
-    RE.md['data_path'] = data_path
-    o_data_path = data_path    # for IOCs on xf16id-ioc1
-
-    if replace_froot is not None:
-        if replace_froot[-1]!="/":
-            replace_froot+="/"
-        data_path = data_path.replace(path, replace_froot)
-        makedirs(data_path, mode=0o0777)
-        #input(f"make sure {data_path} exists on the detector conputer. Hit any key to continue ...")
-
-    if test_only:
-        proc_path = data_path
-    else:
-        proc_path = f"{root_path}/experiments/{current_cycle}/{proposal_id}/{run_id}/"
+
+    #rpath = f"{proposal_id}/{run_id}/"
+    #data_path = f"{data_destination}/{rpath}"
+    # makedirs(data_path, mode=0o0777) this will be created by the IOC?
+    data_path = f"{data_destination}/%s/{current_cycle}/{proposal_id}/{run_id}/"
+    RE.md['data_path'] = data_path   # different IOCs will be writing into subdirectories
+
+    proc_path = f"{proc_destination}/{current_cycle}/{procdir_prefix}{proposal_id}/"
+    check_access(proc_path)
+    proc_path += f"{run_id}/"
     RE.md['proc_path'] = proc_path
     if not os.path.isdir(proc_path):
         makedirs(proc_path, mode=0o2755)
@@ -86,16 +99,13 @@ def login(uname = None, pID = None, rID = None, debug=True, test_only=False,
         elif os.path.isfile(f"{db[-1].start['proc_path']}/exp.h5"):
             shutil.copy(f"{db[-1].start['proc_path']}/exp.h5", f"{proc_path}") 
 
-    if len(share_with)>0:
-        share_dir(proc_path, share_with)
-
     dw,mo,da,tt,yr = time.asctime().split()
     if not os.path.isdir(proc_path+"log"):
         os.mkdir(proc_path+"log")
     logfile = proc_path+("log/%s." % username)+yr+mo+("%02d_" % int(da))+tt.replace(':', '')
     ip = get_ipython()
-    ip.magic("logstop")
-    ip.magic("logstart -ort %s" % logfile)
+    ip.run_line_magic("logstop", "")
+    ip.run_line_magic("logstart", f"-ort {logfile}")
     ip.logger.log_write(f"**LOGIN** {username} @ {time.asctime()}\n")    
 
     if debug:
@@ -106,7 +116,15 @@ def print_time_callback(name, doc):
                 print("#STARTDOC : {}".format(time.ctime(t1)))
         RE.subscribe(print_time_callback)
 
-
+def get_IOC_datapath(ioc_name, substitute_path=None):
+    if data_path=="":
+        print("login first to specify data path:")
+        login()
+    if substitute_path:
+        return data_path.replace(data_destination, substitute_path)%ioc_name
+    else:
+        return data_path%ioc_name
+
 def write_log_msg(msg):
     ip = get_ipython()
     ip.logger.log_write(msg)
@@ -155,12 +173,6 @@ def change_path():
 
     if username==None or proposal_id==None or run_id==None:
         login()
-
-    # to be safe, need to have some kind of lock
-    #get_lock()
-    #if os.path.exists(link_to_data_path):
-    #    os.remove(link_to_data_path)
-    #os.symlink(data_path, link_to_data_path)
 
 
 def logoff(quiet=False):
@@ -189,12 +201,10 @@ def logoff(quiet=False):
     data_path = ""
     proc_path = None
 
-    del RE.md['owner']
-    del RE.md['proposal_id']
-    del RE.md['run_id']      
-    del RE.md['data_path']      
-    del RE.md['proc_path']      
+    for k in ['owner', 'proposal_id', 'run_id', 'data_path', 'proc_path']:
+        if k in RE.md.keys():
+            del RE.md[k]
 
     ip = get_ipython()
-    ip.magic("logstop")
+    ip.run_line_magic("logstop", "")
 

startup/04-sample.py

@@ -1,14 +1,8 @@
 import glob,re
-from enum import Enum
-
-class data_file_path(Enum):
-    old_gpfs = '/GPFS/xf16id/exp_path'
-    gpfs = '/nsls2/xf16id1/data'
-    ramdisk = '/exp_path'
 
 current_sample="test"
 
-def check_sample_name(sample_name, sub_dir=None, check_for_duplicate=True, check_dir=False):    
+def check_sample_name(sample_name, ioc_name="pil1M", sub_dir=None, check_for_duplicate=True, check_dir=False):    
     if len(sample_name)>42:  # file name length limit for Pilatus detectors
         print("Error: the sample name is too long:", len(sample_name))
         return False
@@ -18,7 +12,7 @@ def check_sample_name(sample_name, sub_dir=None, check_for_duplicate=True, check
         return False
 
     if check_for_duplicate:
-        f_path = data_path
+        f_path = data_path%ioc_name
         if sub_dir is not None:
             f_path += ('/'+sub_dir+'/')
         #if PilatusFilePlugin.froot == data_file_path.ramdisk:
@@ -47,7 +41,11 @@ def change_sample(sample_name=None, check_sname=True, exception=True):
     if sample_name is None or sample_name == "":
         sample_name = "test"
     elif check_sname:
-        ret = check_sample_name(sample_name) #, exception)
+        try:
+            ioc_name = pil.active_detectors[0].name
+        except:
+            ioc_name = "pil1M"
+        ret = check_sample_name(sample_name, ioc_name=ioc_name) #, exception)
         if ret==False and exception:
             raise Exception()
 

startup/10-motors.py

@@ -214,7 +214,7 @@ def inverse(self, pos):
 sg1 = SlitsCenterAndGap('XF:16IDC-OP{Slt:G1', name='sg1')
 
 ## Guard Slits 2
-sg2 = SlitsCenterAndGap('XF:16IDC-OP{Slt:G2', name='sg2')
+sg2 = Blades('XF:16IDC-OP{Slt:G2', name='sg2')
 
 #########################################
 ## Detector System

startup/10-vacuum.py

@@ -49,7 +49,7 @@ def pressure(self):
         elif Pr==">1.0E+03":
             P0 = 1.1e3
         else:
-            P0 = np.float(Pr)
+            P0 = float(Pr)
 
         return P0                
 

startup/19-EM_sol_robot.py

@@ -38,11 +38,11 @@ def execute(self,*args):
 
 def waitReady(timeout=-1):    
 	state = PV("SW:State")
-	start = time.clock()
+	start = time.perf_counter()
 	while True:
 		if state.get() not in ("Running","Moving","Busy","Initialize"):
 			break
-		if (timeout>=0) and ((time.clock() - start) > timeout):
+		if (timeout>=0) and ((time.perf_counter() - start) > timeout):
 			raise Exception("Timeout waiting ready")
 		time.sleep(0.01)
 

startup/20-bpm.py

@@ -89,7 +89,5 @@ class NSLS_EM1(NSLS_EM):
 em0.read_attrs = ['x_position', 'y_position']
 
 #tetramm = TetrAMM('XF:16IDC-ES{TETRAMM:1}', name='tetramm')
-
-
 bpm = Best('XF:16IDB-CT{Best}', name='bpm')
 

startup/20-detectors.py

@@ -1,7 +1,3 @@
-#from ophyd import (ProsilicaDetector, SingleTrigger, TIFFPlugin,
-#                   ImagePlugin, DetectorBase, HDF5Plugin, 
-#                   AreaDetector, EpicsSignal, EpicsSignalRO, ROIPlugin,
-#                   TransformPlugin, ProcessPlugin)
 from ophyd.areadetector.detectors import ProsilicaDetector
 from ophyd.areadetector.plugins import ImagePlugin_V34 as ImagePlugin
 from ophyd.areadetector.plugins import TIFFPlugin_V34 as TIFFPlugin
@@ -26,13 +22,10 @@ def make_filename(self):
         return fname, read_path, write_path
 
     def stage(self):
-        global o_data_path,proposal_id,run_id,current_cycle,current_sample
-        # rpath = f"{o_data_path}/tif/"
-        rpath = f"/nsls2/data/lix/legacy/{self.parent.name}/{current_cycle}/{proposal_id}/{run_id}/{current_sample}"
-        #makedirs(rpath, mode=0o0777)
-        #set_and_wait(self.file_path, rpath)
+        global proposal_id, run_id, current_cycle, current_sample
+        rpath = f"{get_IOC_datapath(self.parent.name)}/{current_sample}"
         self.write_path_template = rpath
-        self.create_directory.put(-5)  # create up to 4 levels: ioc, cycle, pid, rid
+        self.create_directory.put(-6)  # create up to 6 levels: ioc, cycle, pid, rid, ???, ??
         super().stage()    
 
 class StandardProsilica(ProsilicaDetector, SingleTrigger):  
@@ -51,7 +44,7 @@ class StandardProsilica(ProsilicaDetector, SingleTrigger):
     stats3 = Cpt(StatsPlugin, 'Stats3:')
     tiff = Cpt(TIFFPluginWithFileStore,
                suffix='TIFF1:',
-               write_path_template='/nsls2/xf16id1/data/',   # this is updated when the plug in is staged
+               write_path_template='/nsls2/xf16id1/data/',   # this is updated when the plugin is staged
                ) 
 
     def __init__(self, *args, detector_id=None, **kwargs):