Merge pull request #66 from usnistgov/update_requirements

Add requirements-all for docs build

AFL/automation/APIServer/APIServer.py

@@ -39,9 +39,17 @@
     import io
     import numpy as np
     from distutils.util import strtobool
-except ImportError:
+except (ModuleNotFoundError,ImportError):
     warnings.warn('Plotting imports failed! Live data plotting will not work on this server.',stacklevel=2)
 
+try:
+    import socket
+    from zeroconf import IPVersion, ServiceInfo, Zeroconf
+    _ADVERTISE_ZEROCONF=True
+except (ModuleNotFoundError,ImportError):
+    warnings.warn('Could not import zeroconf! Network autodiscovery will not work on this server.',stacklevel=2)
+    _ADVERTISE_ZEROCONF=False
+
 class APIServer:
     def __init__(self,name,data = None,experiment='Development',contact='[email protected]',index_template='index.html',new_index_template='index-new.html',plot_template='simple-bokeh.html'):
         self.name = name
@@ -88,16 +96,51 @@ def reset_queue_daemon(self,driver=None):
         self.queue_daemon.terminate()
         self.create_queue(self.driver)
         return 'Success',200
-
+    def advertise_zeroconf(self,**kwargs):
+        if 'port' not in kwargs.keys():
+            port = 5000
+        else:
+            port = kwargs['port']
+        self.zeroconf_info = ServiceInfo(
+            "_aflhttp._tcp.local.",
+            f"{self.queue_daemon.driver.name}._aflhttp._tcp.local.",
+            addresses=[socket.inet_aton("127.0.0.1")],
+            port=port,
+            properties= {
+                        'system_info': 'AFL',
+                        'driver_name': self.queue_daemon.driver.name,
+                        'server_name': self.name,
+                        'contact': self.contact,
+                        'driver_parents': repr(self.queue_daemon.driver.__class__.__mro__)
+                        # other stuff here, AFL system serial, etc.
+                        },
+            server=f"{socket.gethostname()}.local.",
+         )
+        self.zeroconf = Zeroconf(ip_version=IPVersion.All)
+        self.zeroconf.register_service(self.zeroconf_info)
+        print("Started mDNS service advertisement.")
     def run(self,**kwargs):
         if self.queue_daemon is None:
             raise ValueError('create_queue must be called before running server')
-        self.app.run(**kwargs)
+        if _ADVERTISE_ZEROCONF:
+            try:
+                self.advertise_zeroconf(**kwargs)
+            except Exception as e:
+                print(f'failed while trying to start zeroconf {e}, continuing')
+        try:
+            self.app.run(**kwargs)
+        finally:
+            if _ADVERTISE_ZEROCONF:
+                self.zeroconf.unregister_service(self.zeroconf_info)
+                self.zeroconf.close()
 
     def run_threaded(self,start_thread=True,**kwargs):
         if self.queue_daemon is None:
             raise ValueError('create_queue must be called before running server')
-
+        if _ADVERTISE_ZEROCONF:
+            self.advertise_zeroconf(**kwargs)
+
+
         thread = threading.Thread(target=self.app.run,daemon=True,kwargs=kwargs)
         if start_thread:
             thread.start()
@@ -631,4 +674,4 @@ def login_test(self):
     server = APIServer('TestServer')
     server.add_standard_routes()
     server.create_queue(DummyDriver())
-    server.run(host='0.0.0.0',debug=True)
+    server.run(host='0.0.0.0',debug=False)

AFL/automation/APIServer/Client.py

@@ -1,17 +1,20 @@
 import requests,uuid,time,copy,inspect
 from AFL.automation.shared import serialization
-
+from AFL.automation.shared.ServerDiscovery import ServerDiscovery
 
 class Client:
-    '''Communicate with APIServer 
+    '''
+    Communicate with APIServer 
 
-    This class maps pipettor functions to HTTP REST requests that are sent to
-    the server
+    This class provides an interface to generate HTTP REST requests that are sent to
+    an APIServer, monitor the status of those requests, and retrieve the results of
+    those requests.  It is intended to be used as a client to the APIServer class.
     '''
-    def __init__(self,ip=None,port='5000',interactive=False):
-        #trim trailing slash if present
+
+    def __init__(self,ip=None,port='5000',username=None,interactive=False):
         if ip is None:
             raise ValueError('ip (server address) must be specified')
+        #trim trailing slash if present
         if ip[-1] == '/':
             ip = ip[:-1]
         self.ip = ip
@@ -26,6 +29,16 @@ def __init__(self,ip=None,port='5000',interactive=False):
         else:
             #Client.ui = AFL.automation.shared.widgetui.client_construct_ui
             setattr(Client,'ui',AFL.automation.shared.widgetui.client_construct_ui)
+        if username is not None:
+            self.login(username)
+
+
+    @classmethod
+    def from_server_name(cls,server_name,**kwargs):
+        sd = ServerDiscovery()
+        address = ServerDiscovery.sa_discover_server_by_name(server_name)[0]
+        (address,port) = address.split(':')
+        return cls(ip=address,port=port,**kwargs)
 
     def logged_in(self):
         url = self.url + '/login_test'
@@ -36,7 +49,7 @@ def logged_in(self):
             print(response.content)
             return False
 
-    def login(self,username):
+    def login(self,username,populate_commands=True):
         url = self.url + '/login'
         response = requests.post(url,json={'username':username,'password':'domo_arigato'})
         if not (response.status_code == 200):
@@ -45,7 +58,9 @@ def login(self,username):
         # headers should be included in all HTTP requests 
         self.token  = response.json()['token']
         self.headers = {'Authorization':'Bearer {}'.format(self.token)}
-
+        if populate_commands:
+            self.get_queued_commmands()
+            self.get_unqueued_commmands()
 
     def driver_status(self):
         response = requests.get(self.url+'/driver_status',headers=self.headers)
@@ -312,3 +327,9 @@ def get_object(self,name,serialize=True):
         else:
             obj = retval['return_val']
         return obj
+
+    def __str__(self):
+        if self.logged_in():
+            return f'APIServer Client(ip={self.ip},port={self.port}), connected'
+        else:
+            return f'APIServer Client(ip={self.ip},port={self.port}), disconnected'

AFL/automation/APIServer/Driver.py

@@ -13,21 +13,22 @@ def makeRegistrar():
     def registrarfactory(**kwargs):
         #print(f'Set up registrar-factory with registry {registry}...')
         def registrar(func):#,render_hint=None):  #kwarg = kwargs):
-            functions.append(func.__name__)
-            decorator_kwargs[func.__name__]=kwargs
-
-            argspec = inspect.getfullargspec(func)
-            if argspec.defaults is None:
-                fargs = argspec.args
-                fkwargs = []
-            else:
-                fargs = argspec.args[:-len(argspec.defaults)]
-                fkwargs = [(i,j) for i,j in zip(argspec.args[-len(argspec.defaults):],argspec.defaults)]
-            if fargs[0] == 'self':
-                del fargs[0]
-            function_info[func.__name__] = {'args':fargs,'kwargs':fkwargs,'doc':func.__doc__}
-            if 'qb' in kwargs:
-                function_info[func.__name__]['qb'] = kwargs['qb']
+            if func.__name__ not in functions:
+                functions.append(func.__name__)
+                decorator_kwargs[func.__name__]=kwargs
+
+                argspec = inspect.getfullargspec(func)
+                if argspec.defaults is None:
+                    fargs = argspec.args
+                    fkwargs = []
+                else:
+                    fargs = argspec.args[:-len(argspec.defaults)]
+                    fkwargs = [(i,j) for i,j in zip(argspec.args[-len(argspec.defaults):],argspec.defaults)]
+                if fargs[0] == 'self':
+                    del fargs[0]
+                function_info[func.__name__] = {'args':fargs,'kwargs':fkwargs,'doc':func.__doc__}
+                if 'qb' in kwargs:
+                    function_info[func.__name__]['qb'] = kwargs['qb']
             return func  # normally a decorator returns a wrapped function, 
                          # but here we return func unmodified, after registering it
         return registrar
@@ -237,4 +238,4 @@ def deposit_obj(self,obj,uid=None):
             self.dropbox = {}
         self.app.logger.info(f'Storing object in dropbox as {uuid}')
         self.dropbox[uid] = obj
-        return uid
+        return uid

AFL/automation/instrument/APSDNDCAT.py

@@ -332,3 +332,7 @@ def status(self):
         status.append(f'<a href="getReducedData" target="_blank">Live Data (1D)</a>')
         status.append(f'<a href="getReducedData?render_hint=2d_img&reduce_type=2d">Live Data (2D, reduced)</a>')
         return status
+
+if __name__ == '__main__':
+    from AFL.automation.shared.launcher import *
+

AFL/automation/instrument/APSUSAXS.py

@@ -240,3 +240,7 @@ def status(self):
         status.append(f'Next filename: {self.filename}')
         status.append(f'Next project: {self.project}')
         return status
+if __name__ == '__main__':
+    from AFL.automation.shared.launcher import *
+
+

AFL/automation/instrument/CDSAXSLabview.py

@@ -113,7 +113,8 @@ def measureTransmission(self,exp=5,fn='trans',set_empty_transmission=False,retur
  
 
 
-        '''with (LabviewConnection() if lv is None else lv) as lv:
+        '''
+        with (LabviewConnection() if lv is None else lv) as lv:
             self.status_txt = 'Moving beamstop out for transmission...'
             self.moveAxis(self.config['nmc_beamstop_out'],block=True,lv=lv)
             self.moveAxis(self.config['nmc_sample_out'],block=True,lv=lv)
@@ -496,3 +497,6 @@ def __del__(self):
         if(pythoncom._GetInterfaceCount()>0):
             print(f'Closed COM connection, but had remaining objects: {pythoncom._GetInterfaceCount()}')
             SetProcessWorkingSetSize(OpenProcess(PROCESS_ALL_ACCESS,True,GetCurrentProcessId()),-1,-1)
+if __name__ == '__main__':
+    from AFL.automation.shared.launcher import *
+

AFL/automation/instrument/CHESSID3B.py

@@ -355,3 +355,6 @@ def status(self):
         status.append(f'<a href="getReducedData" target="_blank">Live Data (1D)</a>')
         status.append(f'<a href="getReducedData?render_hint=2d_img&reduce_type=2d">Live Data (2D, reduced)</a>')
         return status
+if __name__ == '__main__':
+    from AFL.automation.shared.launcher import *
+

AFL/automation/instrument/DummySAS.py

@@ -1,8 +1,3 @@
-# import win32com
-# import win32com.client
-# from win32process import SetProcessWorkingSetSize
-# from win32api import GetCurrentProcessId,OpenProcess
-# from win32con import PROCESS_ALL_ACCESS
 import gc
 # import pythoncom
 import time
@@ -41,3 +36,6 @@ def expose(self,name=None,exposure=None,nexp=1,block=True,reduce_data=True,measu
     def status(self):
         status = ['Dummy SAS Instrument']
         return status
+
+if __name__ == '__main__':
+    from AFL.automation.shared.launcher import *

AFL/automation/instrument/I22SAXS.py

@@ -161,3 +161,7 @@ def read_integrated(self,scanid):
         self.data.add_array('dI',data['dI'].values.squeeze())
         return scanid
 
+
+_DEFAULT_CUSTOM_PORT = 5001
+if __name__ == '__main__':
+    from AFL.automation.shared.launcher import *

AFL/automation/instrument/ISISLARMOR.py

@@ -9,6 +9,7 @@
 import sans.command_interface.ISISCommandInterface as ici
 # import mantid algorithms, numpy and matplotlib
 from mantid.simpleapi import *
+import numpy as np
 
 from sasdata.dataloader.loader import Loader
 
@@ -34,7 +35,6 @@
 """
 
 PREFIX = "//isis/inst$"
-DATA_LOCATION = "/NDXLARMOR/Instrument/data/cycle_23_5/"
 
 
 class ISISLARMOR(Driver):
@@ -45,7 +45,6 @@ class ISISLARMOR(Driver):
     defaults['sample_thickness'] = 1
 
     defaults['reduced_data_dir'] = './'
-    defaults['tiled_uri'] = 'http://130.246.37.131:8000'
 
     defaults['open_beam_trans_rn'] = -1
     defaults['empty_cell_scatt_rn'] = -1
@@ -54,12 +53,16 @@ class ISISLARMOR(Driver):
     defaults['slow_wait_time'] = 2
     defaults['fast_wait_time'] = 1
     defaults['file_wait_time'] = 1
+
+    defaults['cycle_path'] = "/NDXLARMOR/Instrument/data/cycle_24_2/"
+    defaults['mask_file'] = '/NDXLARMOR/User/Masks/USER_Beaucage_242D_AFL_r86070.TOML'
+
 
     def __init__(self,name:str='ISISLARMOR',overrides=None):
         """ """
         self.app = None
         Driver.__init__(self,name=name,defaults=self.gather_defaults(),overrides=overrides)
-        self.tiled_client = None
+
         self.status_str = "New Server"
 
 
@@ -68,9 +71,6 @@ def status(self):
         status.append(self.status_str)
         return status
 
-    def _init_tiled_client(self):
-        if tiled_client is None:
-            self.tiled_client = from_uri(self.config['tiled_uri'],api_key='NistoRoboto642')
 
     def getRunNumber(self):
         rn = pye.caget("IN:LARMOR:DAE:IRUNNUMBER")
@@ -161,18 +161,23 @@ def waitfor(self, exposure: Number, expose_metric:str):
         else:
             raise ValueError(f'Invalid exposure metric = {expose_metric}')
 
-    def waitforfile(self, fpath, max_t=600):
+    def waitforfile(self, fpath, max_t=900):
         # TODO: Check that the file is not changing, not that is just exists
         self.status_str = f"Waiting for file {fpath} to be written..."
         start_time = time.time()
         while not fpath.exists():
             time.sleep(self.config['file_wait_time'])
             now = time.time()
             if now - start_time >= max_t:
-                raise FileNotFound(f"The file {fpath} was not written within {max_t} seconds.")
-        self.status_str = f"File {fpath} was created successfully."
+                raise FileNotFoundError(f"The file {fpath} was not written within {max_t} seconds.")
+        while not os.access(str(fpath),os.R_OK):
+            time.sleep(self.config['file_wait_time'])
+            now = time.time()
+            if now - start_time >= max_t:
+                raise FileNotFoundError(f"The file {fpath} was not readable within {max_t} seconds.")
+        self.status_str = f"File {fpath} exists and is readable."
 
-    def waitforSASfile(self, fpath, max_t=600):
+    def waitforSASfile(self, fpath, max_t=900):
         # TODO: Check that the file is not changing, not that is just exists
         self.status_str = f"Waiting for file {fpath} to be written..."
         start_time = time.time()
@@ -185,7 +190,7 @@ def waitforSASfile(self, fpath, max_t=600):
                 time.sleep(self.config['file_wait_time'])
                 now = time.time()
                 if now - start_time >= max_t:
-                    raise FileNotFound(f"The file {fpath} was not written within {max_t} seconds.")
+                    raise FileNotFoundError(f"The file {fpath} was not written within {max_t} seconds.")
         self.status_str = f"SASFile {fpath} was loaded successfully."
 
     def waitforsetup(self):
@@ -268,15 +273,21 @@ def expose(
         self.endrun()
 
         if reduce_data:
-            self.waitforfile(Path(PREFIX) / DATA_LOCATION / sampleSANS_fname)
-            self.reduce(name=name, sampleSANS_rn=sampleSANS_rn, sampleTRANS_rn=sampleTRANS_rn)
-    def reduce(self, sampleSANS_rn: int, sampleTRANS_rn: Optional[int]=None, sample_thickness: Number=1, name:str=""):
+            self.waitforfile(Path(PREFIX) / self.config['cycle_path'] / sampleSANS_fname)
+            try:
+                self.reduce(name=name, sampleSANS_rn=sampleSANS_rn, sampleTRANS_rn=sampleTRANS_rn,sample_thickness=self.config['sample_thickness'])
+            except Exception as e:
+                print(f'retrying reduction after an exception {e}, waiting 60 s first for any transient things to resolve')
+                time.sleep(60)
+                self.reduce(name=name, sampleSANS_rn=sampleSANS_rn, sampleTRANS_rn=sampleTRANS_rn,sample_thickness=self.config['sample_thickness'])
+            return self.data['transmission']
+    def reduce(self, sampleSANS_rn: int, sampleTRANS_rn: Optional[int]=None, sample_thickness: Number=2, name:str=""):
         prefix = "//isis/inst$"
         ConfigService.setDataSearchDirs(
             prefix + "/NDXLARMOR/User/Masks/;" + \
-            prefix + "/NDXLARMOR/Instrument/data/cycle_23_5/")
+            prefix + self.config['cycle_path'])
         #mask_file = prefix + '/NDXLARMOR/User/Masks/USER_Beaucage_235C_SampleChanger_r80447.TOML'
-        mask_file = prefix + '/NDXLARMOR/User/Masks/USER_Beaucage_235D_AFL_Robot_r80530.TOML'
+        mask_file = prefix + self.config['mask_file']
         self.status_str = f"Reducing run number {sampleSANS_rn}."
         ici.Clean()
         ici.LARMOR()
@@ -325,11 +336,17 @@ def reduce(self, sampleSANS_rn: int, sampleTRANS_rn: Optional[int]=None, sample_
             warnings.warn("Loaded multiple data from file...taking the last one",stacklevel=2)
 
         sasdata = sasdata[-1]
-
+        self.data['transmission'] = np.mean(sasdata.trans_spectrum[-1].transmission)
         self.data['q'] = sasdata.x
         self.data['filename'] = filename
         self.data.add_array('q',sasdata.x)
         self.data.add_array('I',sasdata.y)
         self.data.add_array('dI',sasdata.dy)
+        self.data.add_array('dq',sasdata.dx)
+
+        self.data.add_array('transmission_spectrum',sasdata.trans_spectrum[-1].transmission)
+        self.data.add_array('transmission_wavelength',sasdata.trans_spectrum[-1].wavelength)
 
 
+if __name__ == '__main__':
+    from AFL.automation.shared.launcher import *