Merge pull request #737 from slaclab/faster_zmq

Use native pickling instead of JSON for zmq interface (Rogue V4)

include/rogue/interfaces/ZmqClient.h

@@ -5,12 +5,12 @@
  * File       : ZmqClient.h
  * Created    : 2019-05-02
  * ----------------------------------------------------------------------------
- * This file is part of the rogue software platform. It is subject to 
- * the license terms in the LICENSE.txt file found in the top-level directory 
- * of this distribution and at: 
- *    https://confluence.slac.stanford.edu/display/ppareg/LICENSE.html. 
- * No part of the rogue software platform, including this file, may be 
- * copied, modified, propagated, or distributed except according to the terms 
+ * This file is part of the rogue software platform. It is subject to
+ * the license terms in the LICENSE.txt file found in the top-level directory
+ * of this distribution and at:
+ *    https://confluence.slac.stanford.edu/display/ppareg/LICENSE.html.
+ * No part of the rogue software platform, including this file, may be
+ * copied, modified, propagated, or distributed except according to the terms
  * contained in the LICENSE.txt file.
  * ----------------------------------------------------------------------------
 **/
@@ -38,7 +38,7 @@ namespace rogue {
             // Zeromq response port
             void * zmqReq_;
 
-            //! Log 
+            //! Log
             std::shared_ptr<rogue::Logging> log_;
 
             uint32_t timeout_;
@@ -62,40 +62,31 @@ namespace rogue {
 
             void setTimeout(uint32_t msecs, bool waitRetry);
 
-            std::string send(std::string value);
-
-            void close();
-
-            virtual void doUpdate (std::string data);
-
-
-            std::string sendWrapper(std::string path, std::string attr, std::string arg, bool rawStr);
-
-            std::string getDisp(std::string path);
-
-            void setDisp(std::string path, std::string value);
+#ifndef NO_PYTHON
+            boost::python::object send(boost::python::object data);
 
-            std::string exec(std::string path, std::string arg = "");
+            virtual void doUpdate (boost::python::object data);
+#endif
 
-            std::string valueDisp(std::string path);
+            void close();
 
       };
       typedef std::shared_ptr<rogue::interfaces::ZmqClient> ZmqClientPtr;
 
 #ifndef NO_PYTHON
 
       //! Stream slave class, wrapper to enable python overload of virtual methods
-      class ZmqClientWrap : 
-         public rogue::interfaces::ZmqClient, 
+      class ZmqClientWrap :
+         public rogue::interfaces::ZmqClient,
          public boost::python::wrapper<rogue::interfaces::ZmqClient> {
 
          public:
 
             ZmqClientWrap (std::string addr, uint16_t port);
 
-            void doUpdate  ( std::string data );
+            void doUpdate  ( boost::python::object data );
 
-            void defDoUpdate  ( std::string data );
+            void defDoUpdate  ( boost::python::object data );
       };
 
       typedef std::shared_ptr<rogue::interfaces::ZmqClientWrap> ZmqClientWrapPtr;

include/rogue/interfaces/ZmqServer.h

@@ -5,12 +5,12 @@
  * File       : ZmqServer.h
  * Created    : 2019-05-02
  * ----------------------------------------------------------------------------
- * This file is part of the rogue software platform. It is subject to 
- * the license terms in the LICENSE.txt file found in the top-level directory 
- * of this distribution and at: 
- *    https://confluence.slac.stanford.edu/display/ppareg/LICENSE.html. 
- * No part of the rogue software platform, including this file, may be 
- * copied, modified, propagated, or distributed except according to the terms 
+ * This file is part of the rogue software platform. It is subject to
+ * the license terms in the LICENSE.txt file found in the top-level directory
+ * of this distribution and at:
+ *    https://confluence.slac.stanford.edu/display/ppareg/LICENSE.html.
+ * No part of the rogue software platform, including this file, may be
+ * copied, modified, propagated, or distributed except according to the terms
  * contained in the LICENSE.txt file.
  * ----------------------------------------------------------------------------
 **/
@@ -44,7 +44,7 @@ namespace rogue {
             std::string addr_;
             uint16_t basePort_;
 
-            //! Log 
+            //! Log
             std::shared_ptr<rogue::Logging> log_;
 
             void runThread();
@@ -61,9 +61,11 @@ namespace rogue {
             ZmqServer (std::string addr, uint16_t port);
             virtual ~ZmqServer();
 
-            void publish(std::string value);
+#ifndef NO_PYTHON
+            void publish(boost::python::object data);
 
-            virtual std::string doRequest (std::string data);
+            virtual boost::python::object doRequest (boost::python::object data);
+#endif
 
             uint16_t port();
 
@@ -74,17 +76,17 @@ namespace rogue {
 #ifndef NO_PYTHON
 
       //! Stream slave class, wrapper to enable python overload of virtual methods
-      class ZmqServerWrap : 
-         public rogue::interfaces::ZmqServer, 
+      class ZmqServerWrap :
+         public rogue::interfaces::ZmqServer,
          public boost::python::wrapper<rogue::interfaces::ZmqServer> {
 
          public:
 
             ZmqServerWrap (std::string addr, uint16_t port);
 
-            std::string doRequest ( std::string data );
+            boost::python::object doRequest ( boost::python::object data );
 
-            std::string defDoRequest ( std::string data );
+            boost::python::object defDoRequest ( boost::python::object data );
       };
 
       typedef std::shared_ptr<rogue::interfaces::ZmqServerWrap> ZmqServerWrapPtr;

python/pyrogue/_Root.py

@@ -22,6 +22,7 @@
 import time
 import queue
 import jsonpickle
+import pickle
 import zipfile
 import traceback
 import datetime
@@ -864,7 +865,7 @@ def _updateWorker(self):
 
                 # Send over zmq link
                 if self._zmqServer is not None:
-                    self._zmqServer._publish(jsonpickle.encode(zmq))
+                    self._zmqServer._publish(pickle.dumps(zmq))
 
                 # Init var list
                 uvars = {}

python/pyrogue/_Variable.py

@@ -595,7 +595,6 @@ def __init__(self, *,
         self._bitSize   = bitSize
         self._bitOffset = bitOffset
         self._verify    = verify
-        self._typeStr   = self._base.name
         self._bytes     = int(math.ceil(float(self._bitOffset[-1] + self._bitSize[-1]) / 8.0))
         self._overlapEn = overlapEn
 
@@ -615,7 +614,7 @@ def __init__(self, *,
                               pollInterval=pollInterval,
                               guiGroup=guiGroup)
 
-
+        self._typeStr   = self._base.name
 
 
     @pr.expose

python/pyrogue/interfaces/_SimpleClient.py

@@ -1,7 +1,7 @@
 #-----------------------------------------------------------------------------
 # Title      : PyRogue Simple ZMQ Client for Rogue
 #-----------------------------------------------------------------------------
-# To use in Matlab first you need both the zmq and jsonpickle package in your
+# To use in Matlab first you need the zmq package in your
 # python installation:
 #
 # > pip install zmq
@@ -24,9 +24,9 @@
 # contained in the LICENSE.txt file.
 #-----------------------------------------------------------------------------
 import zmq
-import jsonpickle
 import threading
 
+
 class SimpleClient(object):
 
     def __init__(self, addr="localhost", port=9099, cb=None):
@@ -52,9 +52,7 @@ def _listen(self):
         self._sub.setsockopt(zmq.SUBSCRIBE,"".encode('utf-8'))
 
         while self._runEn:
-            msg = self._sub.recv_string()
-
-            d = jsonpickle.decode(msg)
+            d = self._sub.recv_pyobj()
 
             for k,val in d.items():
                 self._cb(k,val)
@@ -67,8 +65,8 @@ def _remoteAttr(self,path,attr,*args,**kwargs):
                'kwargs':kwargs}
 
         try:
-            self._req.send_string(jsonpickle.encode(msg))
-            resp = jsonpickle.decode(self._req.recv_string())
+            self._req.send_pyobj(msg)
+            resp = self._req.recv_pyobj()
         except Exception as e:
             raise Exception(f"ZMQ Interface Exception: {e}")
 
@@ -97,4 +95,3 @@ def setDisp(self,path,value):
 
     def exec(self,path,arg):
         return self._remoteAttr(path, '__call__', arg)
-

python/pyrogue/interfaces/_Virtual.py

@@ -16,12 +16,13 @@
 import pyrogue as pr
 import zmq
 import rogue.interfaces
+import pickle
 import functools as ft
-import jsonpickle
 import re
 import time
 import threading
 
+
 class VirtualProperty(object):
     def __init__(self, node, attr):
         self._attr = attr
@@ -270,11 +271,8 @@ def _monWorker(self):
 
 
     def _remoteAttr(self, path, attr, *args, **kwargs):
-        snd = { 'path':path, 'attr':attr, 'args':args, 'kwargs':kwargs }
-        y = jsonpickle.encode(snd)
         try:
-            resp = self._send(y)
-            ret = jsonpickle.decode(resp)
+            ret = pickle.loads(self._send(pickle.dumps({ 'path':path, 'attr':attr, 'args':args, 'kwargs':kwargs })))
         except Exception as e:
             raise Exception(f"ZMQ Interface Exception: {e}")
 
@@ -293,7 +291,7 @@ def _doUpdate(self,data):
         if self._root is None:
             return
 
-        d = jsonpickle.decode(data)
+        d = pickle.loads(data)
 
         for k,val in d.items():
             n = self._root.getNode(k,False)

python/pyrogue/interfaces/_ZmqServer.py

@@ -9,41 +9,34 @@
 # copied, modified, propagated, or distributed except according to the terms
 # contained in the LICENSE.txt file.
 #-----------------------------------------------------------------------------
+
 import rogue.interfaces
-import pyrogue
-import jsonpickle
+import pickle
+
 
 class ZmqServer(rogue.interfaces.ZmqServer):
 
     def __init__(self,*,root,addr,port):
         rogue.interfaces.ZmqServer.__init__(self,addr,port)
         self._root = root
 
-    def encode(self,data,rawStr):
-
-        if rawStr and isinstance(data,str):
-            return data
-        else:
-            return jsonpickle.encode(data)
-
     def _doRequest(self,data):
         try:
-            d = jsonpickle.decode(data)
+            d = pickle.loads(data)
 
             path    = d['path']   if 'path'   in d else None
             attr    = d['attr']   if 'attr'   in d else None
             args    = d['args']   if 'args'   in d else ()
             kwargs  = d['kwargs'] if 'kwargs' in d else {}
-            rawStr  = d['rawStr'] if 'rawStr' in d else False
 
             # Special case to get root node
             if path == "__ROOT__":
-                return self.encode(self._root,rawStr=False)
+                return pickle.dumps(self._root)
 
             node = self._root.getNode(path)
 
             if node is None:
-                return self.encode(None,rawStr=False)
+                return pickle.dumps(None)
 
             nAttr = getattr(node, attr)
 
@@ -54,8 +47,7 @@ def _doRequest(self,data):
             else:
                 resp = nAttr
 
-            return self.encode(resp,rawStr=rawStr)
+            return pickle.dumps(resp)
 
         except Exception as msg:
-            return self.encode(msg,rawStr=False)
-
+            return pickle.dumps(msg)