From a1d73904d6fc776797faa218061223a375f373b5 Mon Sep 17 00:00:00 2001 From: Ryan Herbst Date: Thu, 6 Jul 2023 14:33:13 -0700 Subject: [PATCH 01/23] Fully remove Memory block support --- python/pyrogue/_Device.py | 4 - python/pyrogue/_Memory.py | 402 ------------------------------------- python/pyrogue/_Root.py | 4 - python/pyrogue/__init__.py | 1 - 4 files changed, 411 deletions(-) delete mode 100644 python/pyrogue/_Memory.py diff --git a/python/pyrogue/_Device.py b/python/pyrogue/_Device.py index eb59ca721..2450613e1 100644 --- a/python/pyrogue/_Device.py +++ b/python/pyrogue/_Device.py @@ -165,7 +165,6 @@ def __init__(self, *, description='', memBase=None, offset=0, - size=0, hidden=False, groups=None, expand=False, @@ -176,9 +175,6 @@ def __init__(self, *, hubMax=0, guiGroup=None): - if size != 0: - raise pr.NodeError("Size attribute in Device is not supported!") - """Initialize device class""" if name is None: name = self.__class__.__name__ diff --git a/python/pyrogue/_Memory.py b/python/pyrogue/_Memory.py deleted file mode 100644 index a9d95ae29..000000000 --- a/python/pyrogue/_Memory.py +++ /dev/null @@ -1,402 +0,0 @@ -#----------------------------------------------------------------------------- -# This file is part of the rogue software platform. It is subject to -# the license terms in the LICENSE.txt file found in the top-level directory -# of this distribution and at: -# https://confluence.slac.stanford.edu/display/ppareg/LICENSE.html. -# No part of the rogue software platform, including this file, may be -# copied, modified, propagated, or distributed except according to the terms -# contained in the LICENSE.txt file. -#----------------------------------------------------------------------------- -import pyrogue as pr -import rogue.interfaces.memory as rim -from collections import OrderedDict as odict -import collections -import threading - - -class MemoryDevice(pr.Device): - """ - """ - def __init__(self, *, - name=None, - description='', - memBase=None, - offset=0, - size=0, - hidden=False, - groups=None, - expand=True, - enabled=True, - base=pr.UInt, - wordBitSize=32, - stride=4, - verify=True): - """ - Assigns numerical and categorical values to the MemoryDevice class object - - Parameters - ---------- - name : str - (Default value = None) - - description : str - (Default value = '') - - memBase : - (Default value = None) - - offset : int - (Default value = 0) - - size : int - (Default value = 0) - - hidden : bool - (Default value = False) - - groups : str - (Default value = None) - - expand : bool - (Default value = True) - - enabled : bool - (Default value = True) - - base : - - - wordBitSize : - - - stride : - - - verify : int - (Default value = True) - - returns - ------- - - """ - - super().__init__( - name=name, - description=description, - memBase=memBase, - offset=offset, - hidden=hidden, - groups=groups, - expand=expand, - enabled=enabled, - ) - - self._rawSize = size - self._txnLock = threading.RLock() - - if isinstance(base, pr.Model): - self._base = base - else: - self._base = base(wordBitSize) - - self._wordBitSize = wordBitSize - self._stride = stride - self._verify = verify - - self._setValues = odict() - self._wrValues = odict() # parsed from yaml - self._verValues = odict() - self._wrData = odict() # byte arrays written - self._verData = odict() # verify the written data - - - def _buildBlocks(self): - """ - passes the buildBlocks function - """ - pass - - @pr.expose - def set(self, offset, values, write=False): - """ - Parameters - ---------- - offset : - - values : - - write : bool - (Default value = False) - - Returns - ------- - - """ - with self._txnLock: - self._setValues[offset] = values - if write: - self.writeBlocks() - self.verifyBlocks() - self.checkBlocks() - - @pr.expose - def get(self, offset, numWords): - """ - gets offset and numWords parameters - - Parameters - ---------- - offset : - - numWords : - - - Returns - ------- - - """ - with self._txnLock: - #print(f'get() self._wordBitSize={self._wordBitSize}') - data = self._txnChunker(offset=offset, data=None, - base=self._base, stride=self._stride, - wordBitSize=self._wordBitSize, txnType=rim.Read, - numWords=numWords) - self._waitTransaction(0) - self._clearError() - return [self._base.fromBytes(data[i:i+self._stride]) - for i in range(0, len(data), self._stride)] - - - def _setDict(self, d, writeEach, modes,incGroups,excGroups,keys): - """ - - - Parameters - ---------- - d : - - writeEach : - - modes : - - incGroups : - - excGroups : - - keys : - - - Returns - ------- - - """ - # Parse comma separated values at each offset (key) in d - with self._txnLock: - for offset, values in d.items(): - self._setValues[offset] = [self._base.fromString(s) for s in values.split(',')] - - def _getDict(self,modes,incGroups,excGroups,properties): - """ - - - Parameters - ---------- - modes : - - incGroups : - - excGroups : - - - Returns - ------- - - """ - return None - - def writeBlocks(self, force=False, recurse=True, variable=None, checkEach=False): - """ - - - Parameters - ---------- - force : bool - (Default value = False) - recurse : bool - (Default value = True) - variable : str - (Default value = None) - checkEach : bool - (Default value = False) - - Returns - ------- - - """ - if self.enable.get() is not True: - return - - with self._txnLock: - self._wrValues = self._setValues - for offset, values in self._setValues.items(): - self._txnChunker(offset, values, self._base, self._stride, self._wordBitSize, rim.Write, len(values)) - - # clear out setValues when done - self._setValues = odict() - - - def verifyBlocks(self, recurse=True, variable=None, checkEach=False): - """ - If fall listed functions - - Parameters - ---------- - recurse : bool - (Default value = True) - variable : str - (Default value = None) - checkEach : bool - (Default value = False) - - Returns - ------- - - """ - if (not self._verify) or (self.enable.get() is not True): - return - - with self._txnLock: - for offset, ba in self._wrValues.items(): - # _verValues will not be filled until waitTransaction completes - self._verValues[offset] = self._txnChunker(offset, None, self._base, self._stride, self._wordBitSize, txnType=rim.Verify, numWords=len(ba)) - - def checkBlocks(self, recurse=True, variable=None): - """ - Converts the read verify data back into the native type, compares data and verifies it if necessary, then destroys txn maps when finished. - - Parameters - ---------- - recurse : bool - (Default value = True) - - variable : str - (Default value = None) - - Returns - ------- - - """ - with self._txnLock: - # Wait for all txns to complete - self._waitTransaction(0) - - # Error check? - self._clearError() - - # Convert the read verify data back to the native type - # Can't do this until waitTransaction is done - checkValues = odict() - for offset, ba in self._verValues.items(): - checkValues[offset] = [self._base.fromBytes(ba[i:i+self._stride]) - for i in range(0, len(ba), self._stride)] - - # Do verify if necessary - if len(self._verValues) > 0: - # Compare wrData with verData - if checkValues != self._wrValues: - msg = 'Verify error \n' - msg += f'Expected: \n {self._wrValues} \n' - msg += f'Got: \n {checkValues}' - print(msg) - raise MemoryError(name=self.name, address=self.address, msg=msg, size=self._rawSize) - - - # destroy the txn maps when done with verify - self._verValues = odict() - self._wrValues = odict() - - - def readBlocks(self, recurse=True, variable=None, checkEach=False): - """ - - - Parameters - ---------- - recurse : bool - (Default value = True) - variable : str - (Default value = None) - checkEach : bool - (Default value = False) - - Returns - ------- - - """ - pass - - @pr.expose - @property - def size(self): - """ """ - return self._rawSize - - def _txnChunker(self, offset, data, base=pr.UInt, stride=4, wordBitSize=32, txnType=rim.Write, numWords=1): - """ - - - Parameters - ---------- - offset : - - data : - - base : - (Default value = pr.UInt) - stride : int - (Default value = 4) - wordBitSize : int - (Default value = 32) - txnType : - (Default value = rim.Write) - numWords : int - (Default value = 1) - - Returns - ------- - - """ - - if not isinstance(base, pr.Model): - base = base(wordBitSize) - - if offset + (numWords * stride) > self._rawSize: - raise pr.MemoryError(name=self.name, address=offset|self.address, - msg='Raw transaction outside of device size') - - if base.bitSize > stride*8: - raise pr.MemoryError(name=self.name, address=offset|self.address, - msg='Called raw memory access with wordBitSize > stride') - - if txnType == rim.Write or txnType == rim.Post: - if isinstance(data, bytearray): - ldata = data - elif isinstance(data, collections.abc.Iterable): - ldata = b''.join(base.toBytes(word).ljust(stride, b'\0') for word in data) - else: - ldata = base.toBytes(data) - - else: - if data is not None: - ldata = data - else: - ldata = bytearray(numWords*stride) - - with self._txnLock: - for i in range(offset, offset+len(ldata), self._reqMaxAccess()): - sliceOffset = i | self.offset - txnSize = min(self._reqMaxAccess(), len(ldata)-(i-offset)) - #print(f'sliceOffset: {sliceOffset:#x}, ldata: {ldata}, txnSize: {txnSize}, buffOffset: {i-offset}') - self._reqTransaction(sliceOffset, ldata, txnSize, i-offset, txnType) - - return ldata diff --git a/python/pyrogue/_Root.py b/python/pyrogue/_Root.py index d09928762..9027c128f 100644 --- a/python/pyrogue/_Root.py +++ b/python/pyrogue/_Root.py @@ -367,10 +367,6 @@ def start(self): # Get full list of Blocks and Devices with size tmpList = [] for d in self.deviceList: - - if hasattr(d,"size"): - tmpList.append(d) - for b in d._blocks: if isinstance(b, rim.Block): tmpList.append(b) diff --git a/python/pyrogue/__init__.py b/python/pyrogue/__init__.py index cf442d4ce..9180c393f 100644 --- a/python/pyrogue/__init__.py +++ b/python/pyrogue/__init__.py @@ -19,7 +19,6 @@ from pyrogue._Variable import * from pyrogue._Command import * from pyrogue._Device import * -from pyrogue._Memory import * from pyrogue._Root import * from pyrogue._PollQueue import * from pyrogue._Process import * From f139749a6ee2075ffbedfd456cdc01e36983a55f Mon Sep 17 00:00:00 2001 From: Larry Ruckman Date: Mon, 10 Jul 2023 11:28:11 -0700 Subject: [PATCH 02/23] Update petalinux.rst ### Description - Updating for rogue v6 - Switching from `distutils` to `setuptools3` - distutils-common-base.bbclass is deprecated ``` WARNING: rogue.bb: distutils-common-base.bbclass is deprecated, please use setuptools3-base.bbclass instead ``` --- docs/src/installing/petalinux.rst | 28 ++++++++++++++-------------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/docs/src/installing/petalinux.rst b/docs/src/installing/petalinux.rst index d992f93e1..bd567309e 100644 --- a/docs/src/installing/petalinux.rst +++ b/docs/src/installing/petalinux.rst @@ -18,25 +18,26 @@ You will want to replace the file project-spec/meta-user/recipes-apps/rogue/rogu .. code:: - ROGUE_VERSION = "5.18.2" - ROGUE_MD5SUM = "38bf1bc4108eb08fc56ee9017be40c50" - + ROGUE_VERSION = "6.0.0" + ROGUE_MD5SUM = "42d6ffe9894c10a5d0e4c43834878e73" + SUMMARY = "Recipe to build Rogue" HOMEPAGE ="https://github.com/slaclab/rogue" LICENSE = "MIT" LIC_FILES_CHKSUM = "file://${COMMON_LICENSE_DIR}/MIT;md5=0835ade698e0bcf8506ecda2f7b4f302" - + SRC_URI = "https://github.com/slaclab/rogue/archive/v${ROGUE_VERSION}.tar.gz" SRC_URI[md5sum] = "${ROGUE_MD5SUM}" - + S = "${WORKDIR}/rogue-${ROGUE_VERSION}" PROVIDES = "rogue" EXTRA_OECMAKE += "-DROGUE_INSTALL=system -DROGUE_VERSION=v${ROGUE_VERSION}" - - inherit cmake python3native distutils3 - + + inherit cmake python3native setuptools3 + DEPENDS += " \ python3 \ + python3-setuptools \ python3-native \ python3-numpy \ python3-numpy-native \ @@ -51,7 +52,7 @@ You will want to replace the file project-spec/meta-user/recipes-apps/rogue/rogu boost \ cmake \ " - + RDEPENDS:${PN} += " \ python3-numpy \ python3-pyzmq \ @@ -63,22 +64,21 @@ You will want to replace the file project-spec/meta-user/recipes-apps/rogue/rogu python3-json \ python3-logging \ " - + FILES:${PN}-dev += "/usr/include/rogue/*" FILES:${PN} += "/usr/lib/*" - + do_configure() { cmake_do_configure bbplain $(cp -vH ${WORKDIR}/build/setup.py ${S}/.) + bbplain $(sed -i "s/..\/python/python/" ${S}/setup.py) } - + do_install() { cmake_do_install distutils3_do_install } - - Update the ROGUE_VERSION line for an updated version when appropriate (min version is 5.6.1). You will need to first download the tar.gz file and compute the MD5SUM using the following commands if you update the ROGUE_VERSION line: .. code:: From f1fa764e353943b52c7cd911b06876315509255a Mon Sep 17 00:00:00 2001 From: Larry Ruckman Date: Wed, 12 Jul 2023 08:21:00 -0700 Subject: [PATCH 03/23] Update anaconda.rst for rogue v6 --- docs/src/installing/anaconda.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/src/installing/anaconda.rst b/docs/src/installing/anaconda.rst index adf7001b8..4b6112f95 100644 --- a/docs/src/installing/anaconda.rst +++ b/docs/src/installing/anaconda.rst @@ -63,7 +63,7 @@ Alternatively you can install a specific released version of Rogue: .. code:: - $ conda create -n rogue_v5.18.4 -c conda-forge -c tidair-tag rogue=v5.18.4 + $ conda create -n rogue_v6.0.0 -c conda-forge -c tidair-tag rogue=v6.0.0 Using Rogue In Anaconda ======================= @@ -76,7 +76,7 @@ To activate: $ conda activate rogue_tag -Replace rogue_tag with the name you used when creating your environment (rogue_dev or rogue_5.8.0). +Replace rogue_tag with the name you used when creating your environment (rogue_dev or rogue_v6.0.0). To deactivate: From 4e417f0db957557d721d9b83c7c00d40f951a5bd Mon Sep 17 00:00:00 2001 From: Larry Ruckman Date: Wed, 12 Jul 2023 12:44:40 -0700 Subject: [PATCH 04/23] Update petalinux.rst --- docs/src/installing/petalinux.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/src/installing/petalinux.rst b/docs/src/installing/petalinux.rst index bd567309e..52d1b6db2 100644 --- a/docs/src/installing/petalinux.rst +++ b/docs/src/installing/petalinux.rst @@ -76,7 +76,7 @@ You will want to replace the file project-spec/meta-user/recipes-apps/rogue/rogu do_install() { cmake_do_install - distutils3_do_install + setuptools3_do_install } Update the ROGUE_VERSION line for an updated version when appropriate (min version is 5.6.1). You will need to first download the tar.gz file and compute the MD5SUM using the following commands if you update the ROGUE_VERSION line: From c16e7c8814d4d42151454cc39eb89a6c5787fa09 Mon Sep 17 00:00:00 2001 From: Larry Ruckman Date: Fri, 14 Jul 2023 14:14:55 -0700 Subject: [PATCH 05/23] Update petalinux.rst --- docs/src/installing/petalinux.rst | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/docs/src/installing/petalinux.rst b/docs/src/installing/petalinux.rst index 52d1b6db2..b7ad42d37 100644 --- a/docs/src/installing/petalinux.rst +++ b/docs/src/installing/petalinux.rst @@ -32,12 +32,12 @@ You will want to replace the file project-spec/meta-user/recipes-apps/rogue/rogu S = "${WORKDIR}/rogue-${ROGUE_VERSION}" PROVIDES = "rogue" EXTRA_OECMAKE += "-DROGUE_INSTALL=system -DROGUE_VERSION=v${ROGUE_VERSION}" - - inherit cmake python3native setuptools3 + + # Note: distutils3 is depreciated in petalinux 2023.1 and need to switch to setuptools3 + inherit cmake python3native distutils3 DEPENDS += " \ python3 \ - python3-setuptools \ python3-native \ python3-numpy \ python3-numpy-native \ @@ -68,15 +68,14 @@ You will want to replace the file project-spec/meta-user/recipes-apps/rogue/rogu FILES:${PN}-dev += "/usr/include/rogue/*" FILES:${PN} += "/usr/lib/*" - do_configure() { + do_configure:prepend() { cmake_do_configure bbplain $(cp -vH ${WORKDIR}/build/setup.py ${S}/.) bbplain $(sed -i "s/..\/python/python/" ${S}/setup.py) } - do_install() { + do_install:prepend() { cmake_do_install - setuptools3_do_install } Update the ROGUE_VERSION line for an updated version when appropriate (min version is 5.6.1). You will need to first download the tar.gz file and compute the MD5SUM using the following commands if you update the ROGUE_VERSION line: From 4244d589bc5f0284033f17ddc4fe3f77ed27f3a5 Mon Sep 17 00:00:00 2001 From: Larry Ruckman Date: Fri, 14 Jul 2023 14:17:38 -0700 Subject: [PATCH 06/23] Update anaconda.rst --- docs/src/installing/anaconda.rst | 10 +--------- 1 file changed, 1 insertion(+), 9 deletions(-) diff --git a/docs/src/installing/anaconda.rst b/docs/src/installing/anaconda.rst index 4b6112f95..b9b0efe66 100644 --- a/docs/src/installing/anaconda.rst +++ b/docs/src/installing/anaconda.rst @@ -53,12 +53,6 @@ If you already have an anaconda environment that you would like to install Rogue $ conda install -c tidair-tag -c conda-forge rogue -The above commands will install the latest version of Rogue from the `main` branch. If you want to install the `pre-release` version of Rogue, run the following: - -.. code:: - - $ conda create -n rogue_dev -c tidair-dev -c conda-forge rogue - Alternatively you can install a specific released version of Rogue: .. code:: @@ -76,7 +70,7 @@ To activate: $ conda activate rogue_tag -Replace rogue_tag with the name you used when creating your environment (rogue_dev or rogue_v6.0.0). +Replace rogue_tag with the name you used when creating your environment (e.g. rogue_v6.0.0). To deactivate: @@ -100,8 +94,6 @@ If you want to update Rogue, run the following command after activating the Rogu $ conda update rogue -c tidair-tag -Replace tidair-tag with tidair-dev for pre-release - Deleting Anaconda Environment ============================= From d8b41dcad6328ff9ea0b9984dba000f7e69ac1e5 Mon Sep 17 00:00:00 2001 From: Ryan Herbst Date: Sun, 6 Aug 2023 23:29:13 -0700 Subject: [PATCH 07/23] Reset fifo counters when countReset chain is called --- python/pyrogue/interfaces/stream/_Fifo.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/python/pyrogue/interfaces/stream/_Fifo.py b/python/pyrogue/interfaces/stream/_Fifo.py index bcd291fcb..d195479b8 100644 --- a/python/pyrogue/interfaces/stream/_Fifo.py +++ b/python/pyrogue/interfaces/stream/_Fifo.py @@ -67,3 +67,7 @@ def __lshift__(self,other): def __rshift__(self,other): pyrogue.streamConnect(self,other) return other + + def countReset(self): + self._fifo.clearCnt() + From 0fa5a6a026c0865a747ed45b663848f7c0e98b73 Mon Sep 17 00:00:00 2001 From: Larry Ruckman Date: Mon, 7 Aug 2023 10:22:17 -0700 Subject: [PATCH 08/23] flake8 bug fixes --- python/pyrogue/_HelperFunctions.py | 6 +++--- python/pyrogue/_Variable.py | 2 +- python/pyrogue/interfaces/stream/_Fifo.py | 1 - 3 files changed, 4 insertions(+), 5 deletions(-) diff --git a/python/pyrogue/_HelperFunctions.py b/python/pyrogue/_HelperFunctions.py index bf5b77761..812c82a2f 100644 --- a/python/pyrogue/_HelperFunctions.py +++ b/python/pyrogue/_HelperFunctions.py @@ -349,13 +349,13 @@ def _var_representer(dumper, data): ------- """ - if type(data.value) == bool: + if isinstance(type(data.value), bool): enc = 'tag:yaml.org,2002:bool' elif data.enum is not None: enc = 'tag:yaml.org,2002:str' - elif type(data.value) == int: + elif isinstance(type(data.value), int): enc = 'tag:yaml.org,2002:int' - elif type(data.value) == float: + elif isinstance(type(data.value), float): enc = 'tag:yaml.org,2002:float' else: enc = 'tag:yaml.org,2002:str' diff --git a/python/pyrogue/_Variable.py b/python/pyrogue/_Variable.py index 4d22400d5..17df32573 100644 --- a/python/pyrogue/_Variable.py +++ b/python/pyrogue/_Variable.py @@ -214,7 +214,7 @@ def __init__(self, *, self._enum = disp elif isinstance(disp, list): self._enum = {k:str(k) for k in disp} - elif type(value) == bool and enum is None: + elif isinstance(type(value), bool) and enum is None: self._enum = {False: 'False', True: 'True'} if self._enum is not None: diff --git a/python/pyrogue/interfaces/stream/_Fifo.py b/python/pyrogue/interfaces/stream/_Fifo.py index d195479b8..bcd3a6e1b 100644 --- a/python/pyrogue/interfaces/stream/_Fifo.py +++ b/python/pyrogue/interfaces/stream/_Fifo.py @@ -70,4 +70,3 @@ def __rshift__(self,other): def countReset(self): self._fifo.clearCnt() - From ffd1f6c24461dcec1e485238b4742b3be5335ef3 Mon Sep 17 00:00:00 2001 From: Larry Ruckman Date: Mon, 7 Aug 2023 10:22:17 -0700 Subject: [PATCH 09/23] flake8 bug fixes --- python/pyrogue/_HelperFunctions.py | 6 +++--- python/pyrogue/_Variable.py | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/python/pyrogue/_HelperFunctions.py b/python/pyrogue/_HelperFunctions.py index bf5b77761..812c82a2f 100644 --- a/python/pyrogue/_HelperFunctions.py +++ b/python/pyrogue/_HelperFunctions.py @@ -349,13 +349,13 @@ def _var_representer(dumper, data): ------- """ - if type(data.value) == bool: + if isinstance(type(data.value), bool): enc = 'tag:yaml.org,2002:bool' elif data.enum is not None: enc = 'tag:yaml.org,2002:str' - elif type(data.value) == int: + elif isinstance(type(data.value), int): enc = 'tag:yaml.org,2002:int' - elif type(data.value) == float: + elif isinstance(type(data.value), float): enc = 'tag:yaml.org,2002:float' else: enc = 'tag:yaml.org,2002:str' diff --git a/python/pyrogue/_Variable.py b/python/pyrogue/_Variable.py index 4d22400d5..17df32573 100644 --- a/python/pyrogue/_Variable.py +++ b/python/pyrogue/_Variable.py @@ -214,7 +214,7 @@ def __init__(self, *, self._enum = disp elif isinstance(disp, list): self._enum = {k:str(k) for k in disp} - elif type(value) == bool and enum is None: + elif isinstance(type(value), bool) and enum is None: self._enum = {False: 'False', True: 'True'} if self._enum is not None: From fa79e3fb5757355055db3d4f124141cd48a14a2f Mon Sep 17 00:00:00 2001 From: Ryan Herbst Date: Mon, 7 Aug 2023 19:11:42 -0700 Subject: [PATCH 10/23] Fix error in isinstance call --- python/pyrogue/_HelperFunctions.py | 6 +++--- python/pyrogue/_Variable.py | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/python/pyrogue/_HelperFunctions.py b/python/pyrogue/_HelperFunctions.py index 812c82a2f..7fb2df430 100644 --- a/python/pyrogue/_HelperFunctions.py +++ b/python/pyrogue/_HelperFunctions.py @@ -349,13 +349,13 @@ def _var_representer(dumper, data): ------- """ - if isinstance(type(data.value), bool): + if isinstance(data.value, bool): enc = 'tag:yaml.org,2002:bool' elif data.enum is not None: enc = 'tag:yaml.org,2002:str' - elif isinstance(type(data.value), int): + elif isinstance(data.value, int): enc = 'tag:yaml.org,2002:int' - elif isinstance(type(data.value), float): + elif isinstance(data.value, float): enc = 'tag:yaml.org,2002:float' else: enc = 'tag:yaml.org,2002:str' diff --git a/python/pyrogue/_Variable.py b/python/pyrogue/_Variable.py index 17df32573..973aaeb72 100644 --- a/python/pyrogue/_Variable.py +++ b/python/pyrogue/_Variable.py @@ -214,7 +214,7 @@ def __init__(self, *, self._enum = disp elif isinstance(disp, list): self._enum = {k:str(k) for k in disp} - elif isinstance(type(value), bool) and enum is None: + elif isinstance(value, bool) and enum is None: self._enum = {False: 'False', True: 'True'} if self._enum is not None: From 3adc289cb7c7b3b484aab56537e359481eca6ce7 Mon Sep 17 00:00:00 2001 From: Ryan Herbst Date: Mon, 7 Aug 2023 19:27:57 -0700 Subject: [PATCH 11/23] REmove C++ Header Generation --- python/pyrogue/_Root.py | 23 ++++------------------- 1 file changed, 4 insertions(+), 19 deletions(-) diff --git a/python/pyrogue/_Root.py b/python/pyrogue/_Root.py index 9027c128f..581b39c05 100644 --- a/python/pyrogue/_Root.py +++ b/python/pyrogue/_Root.py @@ -555,7 +555,7 @@ def getNode(self,path): return obj @pr.expose - def saveAddressMap(self,fname,headerEn=False): + def saveAddressMap(self,fname): """ @@ -572,7 +572,6 @@ def saveAddressMap(self,fname,headerEn=False): """ # First form header - # Changing these names here requires changing the createVariable() method in LibraryBase.cpp header = "Path\t" header += "TypeStr\t" header += "Address\t" @@ -637,23 +636,9 @@ def saveAddressMap(self,fname,headerEn=False): lines.append(data) with open(fname,'w') as f: - - if headerEn: - f.write('// #############################################\n') - f.write('// Auto Generated Header File From Rogue\n') - f.write('// #############################################\n') - f.write('#ifndef __ROGUE_ADDR_MAP_H__\n') - f.write('#define __ROGUE_ADDR_MAP_H__\n\n') - f.write(f'#define ROGUE_ADDR_MAP "{header}|"\\\n') - - for line in lines: - f.write(f' "{line}|"\\\n') - - f.write('\n#endif\n') - else: - f.write(header + '\n') - for line in lines: - f.write(line + '\n') + f.write(header + '\n') + for line in lines: + f.write(line + '\n') @pr.expose def saveVariableList(self,fname,polledOnly=False,incGroups=None): From 759b180408fe5d102f959ff9e3c6e8a263dd8ce7 Mon Sep 17 00:00:00 2001 From: Ryan Herbst Date: Mon, 7 Aug 2023 19:42:07 -0700 Subject: [PATCH 12/23] Remove LibraryBase --- include/rogue/LibraryBase.h | 119 ------------ src/rogue/CMakeLists.txt | 1 - src/rogue/LibraryBase.cpp | 351 ------------------------------------ 3 files changed, 471 deletions(-) delete mode 100644 include/rogue/LibraryBase.h delete mode 100644 src/rogue/LibraryBase.cpp diff --git a/include/rogue/LibraryBase.h b/include/rogue/LibraryBase.h deleted file mode 100644 index 69082ea53..000000000 --- a/include/rogue/LibraryBase.h +++ /dev/null @@ -1,119 +0,0 @@ -/** - *----------------------------------------------------------------------------- - * Title : Rogue as a library base class - * ---------------------------------------------------------------------------- - * File : LibraryBase.h - * ---------------------------------------------------------------------------- - * Description: - * Base class for creating a Rogue shared library - * ---------------------------------------------------------------------------- - * This file is part of the rogue software platform. It is subject to - * the license terms in the LICENSE.txt file found in the top-level directory - * of this distribution and at: - * https://confluence.slac.stanford.edu/display/ppareg/LICENSE.html. - * No part of the rogue software platform, including this file, may be - * copied, modified, propagated, or distributed except according to the terms - * contained in the LICENSE.txt file. - * ---------------------------------------------------------------------------- - **/ -#ifndef __ROGUE_LIBRARY_BASE_H__ -#define __ROGUE_LIBRARY_BASE_H__ -#include "rogue/Directives.h" - -#include - -#include -#include -#include -#include -#include -#include -#include - -#include "rogue/Logging.h" -#include "rogue/interfaces/memory/Block.h" -#include "rogue/interfaces/memory/Slave.h" -#include "rogue/interfaces/memory/Variable.h" -#include "rogue/interfaces/stream/Master.h" -#include "rogue/interfaces/stream/Slave.h" - -namespace rogue { - -//! LibraryBase -class LibraryBase { - // Logger - std::shared_ptr log_; - - //! Map of variables - std::map> variables_; - - //! Map of blocks by block name - std::map> blocks_; - - //! Map of memory interfaces - std::map> memSlaves_; - - //! Map of missing memory interfaces - std::set memSlavesMissing_; - - public: - LibraryBase(); - ~LibraryBase(); - - //! Add slave memory interface - void addMemory(std::string name, std::shared_ptr slave); - - // Parse memory map - void parseMemMap(std::string map); - void parseMemMap(const char* filePath, char delim = '\n'); - void parseMemMap(std::istream& fStream, char delim); - - static std::shared_ptr create(); - - //! Read all variables - void readAll(); - - //! Get variable by name - std::shared_ptr getVariable(const std::string& name); - - //! Get a const reference to the map of variables - const std::map>& getVariableList(); - - //! Get block by name - std::shared_ptr getBlock(const std::string& name); - - //! Get a map of blocks - const std::map> getBlockList(); - - private: - //! Create a variable - void createVariable( - std::map& data, - std::map>>& blockVars); - - //! Helper function to get string from fields - std::string getFieldString(std::map fields, std::string name); - - //! Helper function to get uint64_t from fields - uint64_t getFieldUInt64(std::map fields, std::string name); - - //! Helper function to get uint32_t from fields - uint32_t getFieldUInt32(std::map fields, std::string name, bool def = false); - - //! Helper function to get bool from fields - bool getFieldBool(std::map fields, std::string name); - - //! Helper function to get double from fields - double getFieldDouble(std::map fields, std::string name); - - //! Helper function to get uint32_t vector from fields - std::vector getFieldVectorUInt32(std::map fields, std::string name); - - //! Dump the current state of the registers in the system - void dumpRegisterStatus(std::string filename, bool read, bool includeStatus = false); -}; - -typedef std::shared_ptr LibraryBasePtr; -} // namespace rogue - -#endif diff --git a/src/rogue/CMakeLists.txt b/src/rogue/CMakeLists.txt index 767b6bddb..b0dcb98d4 100644 --- a/src/rogue/CMakeLists.txt +++ b/src/rogue/CMakeLists.txt @@ -23,7 +23,6 @@ target_sources(rogue-core PRIVATE "${CMAKE_CURRENT_LIST_DIR}/GeneralError.cpp") target_sources(rogue-core PRIVATE "${CMAKE_CURRENT_LIST_DIR}/GilRelease.cpp") target_sources(rogue-core PRIVATE "${CMAKE_CURRENT_LIST_DIR}/Logging.cpp") target_sources(rogue-core PRIVATE "${CMAKE_CURRENT_LIST_DIR}/ScopedGil.cpp") -target_sources(rogue-core PRIVATE "${CMAKE_CURRENT_LIST_DIR}/LibraryBase.cpp") target_sources(rogue-core PRIVATE "${CMAKE_CURRENT_LIST_DIR}/Version.cpp") if (NOT NO_PYTHON) diff --git a/src/rogue/LibraryBase.cpp b/src/rogue/LibraryBase.cpp deleted file mode 100644 index 1346b5e31..000000000 --- a/src/rogue/LibraryBase.cpp +++ /dev/null @@ -1,351 +0,0 @@ -/** - *----------------------------------------------------------------------------- - * Title : Rogue as a library base class - * ---------------------------------------------------------------------------- - * File : LibraryBase.cpp - * ---------------------------------------------------------------------------- - * Description: - * Base class for creating a Rogue shared library - * ---------------------------------------------------------------------------- - * This file is part of the rogue software platform. It is subject to - * the license terms in the LICENSE.txt file found in the top-level directory - * of this distribution and at: - * https://confluence.slac.stanford.edu/display/ppareg/LICENSE.html. - * No part of the rogue software platform, including this file, may be - * copied, modified, propagated, or distributed except according to the terms - * contained in the LICENSE.txt file. - * ---------------------------------------------------------------------------- - **/ - -#include "rogue/LibraryBase.h" - -#include - -#include -#include -#include -#include -#include -#include -#include - -#include "rogue/GeneralError.h" -#include "rogue/interfaces/memory/Constants.h" - -namespace ris = rogue::interfaces::stream; -namespace rim = rogue::interfaces::memory; - -rogue::LibraryBasePtr rogue::LibraryBase::create() { - rogue::LibraryBasePtr ret = std::make_shared(); - return (ret); -} - -rogue::LibraryBase::LibraryBase() { - log_ = rogue::Logging::create("LibraryBase"); -} - -rogue::LibraryBase::~LibraryBase() {} - -//! Add slave memory interface -void rogue::LibraryBase::addMemory(std::string name, rim::SlavePtr slave) { - memSlaves_[name] = slave; -} - -//! Get variable by name -rim::VariablePtr rogue::LibraryBase::getVariable(const std::string& name) { - return variables_[name]; -} - -//! Get a map of variables -const std::map& rogue::LibraryBase::getVariableList() { - return variables_; -} - -//! Get block by name -rim::BlockPtr rogue::LibraryBase::getBlock(const std::string& name) { - return blocks_[name]; -} - -//! Get a map of blocks -const std::map rogue::LibraryBase::getBlockList() { - return blocks_; -} - -// Parse memory map -void rogue::LibraryBase::parseMemMap(std::string map) { - // Determine which delimiter we are using - char delim; - std::size_t found = map.find("|"); - if (found != std::string::npos) - delim = '|'; - else - delim = '\n'; - std::istringstream fStream(map); - parseMemMap(fStream, delim); -} - -void rogue::LibraryBase::parseMemMap(const char* filePath, char delim) { - std::ifstream fStream(filePath); - parseMemMap(fStream, delim); -} - -void rogue::LibraryBase::parseMemMap(std::istream& fStream, char delim) { - std::string line; - std::string field; - std::vector key; - - std::map> blockVars; - std::map>::iterator it; - - uint32_t x; - bool doKey = true; - - while (std::getline(fStream, line, delim)) { - std::istringstream lStream(line); - std::map fields; - x = 0; - - while (std::getline(lStream, field, '\t')) { - if (doKey) - key.push_back(field); - else - fields.insert(std::pair(key[x], field)); - ++x; - } - - if (!doKey) createVariable(fields, blockVars); - doKey = false; - } - - // Add variables to the blocks - for (it = blockVars.begin(); it != blockVars.end(); ++it) { - rim::BlockPtr blk = blocks_[it->first]; - blk->addVariables(it->second); - } -} - -//! Create a variable -// The fields used here are defined in saveAddressMap in _Root.py -void rogue::LibraryBase::createVariable(std::map& data, - std::map>& blockVars) { - // Extract fields - std::string name = getFieldString(data, "Path"); - std::string mode = getFieldString(data, "Mode"); - std::string mbName = getFieldString(data, "MemBaseName"); - std::string blkName = getFieldString(data, "BlockName"); - - // Verify memory slave exists - if (memSlaves_.find(mbName) == memSlaves_.end()) { - if (memSlavesMissing_.find(mbName) != memSlavesMissing_.end()) { - // Just return as we've already reported this. - return; - } - memSlavesMissing_.insert(mbName); - log_->info("LibraryBase::createVariable: '%s' memory interface '%s' not found!", name.c_str(), mbName.c_str()); - return; - } - - bool overlapEn = getFieldBool(data, "OverlapEn"); - bool verify = getFieldBool(data, "Verify"); - bool bulkEn = getFieldBool(data, "BulkEn"); - bool updateNotify = getFieldBool(data, "UpdateNotify"); - bool byteReverse = getFieldBool(data, "ByteReverse"); - bool bitReverse = getFieldBool(data, "BitReverse"); - - uint64_t offset = getFieldUInt64(data, "Address"); - uint32_t modelId = getFieldUInt32(data, "ModelId"); - uint32_t binPoint = getFieldUInt32(data, "BinPoint"); - uint32_t blockSize = getFieldUInt32(data, "BlockSize"); - uint32_t numValues = getFieldUInt32(data, "NumValues", true); - uint32_t valueBits = getFieldUInt32(data, "ValueBits", true); - uint32_t valueStride = getFieldUInt32(data, "ValueStride", true); - uint32_t retryCount = getFieldUInt32(data, "RetryCount", true); - - double minimum = getFieldDouble(data, "Minimum"); - double maximum = getFieldDouble(data, "Maximum"); - - std::vector bitOffset = getFieldVectorUInt32(data, "BitOffset"); - std::vector bitSize = getFieldVectorUInt32(data, "BitSize"); - - // Create the holding block if it does not already exist - if (blocks_.find(blkName) == blocks_.end()) { - rim::BlockPtr block = rim::Block::create(offset, blockSize); - blocks_.insert(std::pair(blkName, block)); - std::vector vp; - blockVars.insert(std::pair>(blkName, vp)); - - // Connect to memory slave - *block >> memSlaves_[mbName]; - - // Enable the block - block->setEnable(true); - } - - // Create variable - rim::VariablePtr var = rim::Variable::create(name, - mode, - minimum, - maximum, - offset, - bitOffset, - bitSize, - overlapEn, - verify, - bulkEn, - updateNotify, - modelId, - byteReverse, - bitReverse, - binPoint, - numValues, - valueBits, - valueStride, - retryCount); - - // Adjust min transaction size to match blocksize field - var->shiftOffsetDown(0, blockSize); - - // Add to block list - blockVars[blkName].push_back(var); - variables_[name] = var; -} - -// Read all variables -void rogue::LibraryBase::readAll() { - std::map::iterator it; - for (it = variables_.begin(); it != variables_.end(); ++it) { it->second->read(); } -} - -//! Helper function to get string from fields -std::string rogue::LibraryBase::getFieldString(std::map fields, std::string name) { - if (fields.find(name) == fields.end()) - throw(rogue::GeneralError::create("LibraryBase::getFieldString", - "Failed to find field '%s' in exported address map", - name.c_str())); - - return fields[name]; -} - -//! Helper function to get uint64_t from fields -uint64_t rogue::LibraryBase::getFieldUInt64(std::map fields, std::string name) { - uint64_t ret; - - if (fields.find(name) == fields.end()) - throw(rogue::GeneralError::create("LibraryBase::getFieldUInt64", - "Failed to find field '%s' in exported address map", - name.c_str())); - - if (fields[name] == "None") return 0; - - try { - ret = std::stol(fields[name], 0, 0); - } catch (...) { - throw(rogue::GeneralError::create("LibraryBase::getFieldUInt64", - "Failed to extract uint64_t from field '%s' in exported address map", - name.c_str())); - } - return ret; -} - -//! Helper function to get uint32_t from fields -uint32_t rogue::LibraryBase::getFieldUInt32(std::map fields, std::string name, bool def) { - uint32_t ret; - - if (fields.find(name) == fields.end()) { - if (def) - return 0; - else - throw(rogue::GeneralError::create("LibraryBase::getFieldUInt32", - "Failed to find field '%s' in exported address map", - name.c_str())); - } - - if (fields[name] == "None") return 0; - - try { - ret = std::stoi(fields[name], 0, 0); - } catch (...) { - throw(rogue::GeneralError::create("LibraryBase::getFieldUInt32", - "Failed to extract uint32_t from field '%s' in exported address map", - name.c_str())); - } - return ret; -} - -//! Helper function to get bool from fields -bool rogue::LibraryBase::getFieldBool(std::map fields, std::string name) { - if (fields.find(name) == fields.end()) - throw(rogue::GeneralError::create("LibraryBase::getFieldBool", - "Failed to find field '%s' in exported address map", - name.c_str())); - - return (fields[name] == "True"); -} - -//! Helper function to get double from fields -double rogue::LibraryBase::getFieldDouble(std::map fields, std::string name) { - double ret; - - if (fields.find(name) == fields.end()) - throw(rogue::GeneralError::create("LibraryBase::getFieldDouble", - "Failed to find field '%s' in exported address map", - name.c_str())); - - if (fields[name] == "None") return 0.0; - - try { - ret = std::stod(fields[name]); - } catch (...) { - throw(rogue::GeneralError::create("LibraryBase::getFieldDouble", - "Failed to extract double from field '%s' in exported address map", - name.c_str())); - } - return ret; -} - -//! Helper function to get uint32_t vector from fields -std::vector rogue::LibraryBase::getFieldVectorUInt32(std::map fields, - std::string name) { - std::vector ret; - std::string fld; - std::string sub; - uint32_t val; - - if (fields.find(name) == fields.end()) - throw(rogue::GeneralError::create("LibraryBase::getFieldVector32", - "Failed to find field '%s' in exported address map", - name.c_str())); - - fld = fields[name]; - - std::replace(fld.begin(), fld.end(), '[', ' '); - std::replace(fld.begin(), fld.end(), ']', ','); - - std::istringstream fStream(fld); - - while (std::getline(fStream, sub, ',')) { - try { - val = std::stoi(sub); - } catch (...) { - throw(rogue::GeneralError::create("LibraryBase::getFieldVectorUInt32", - "Failed to extract uint32 from field '%s' in exported address map", - name.c_str())); - } - ret.push_back(val); - } - return ret; -} - -//! Dump the current state of the registers in the system -void rogue::LibraryBase::dumpRegisterStatus(std::string filename, bool read, bool includeStatus) { - std::map::iterator it; - - std::ofstream myfile; - - myfile.open(filename); - - for (it = variables_.begin(); it != variables_.end(); ++it) { - if (includeStatus || it->second->mode() != "RO") { myfile << it->second->getDumpValue(read); } - } - myfile.close(); -} From 5962462c12791d94c583e580a0c0438363e175df Mon Sep 17 00:00:00 2001 From: Ryan Herbst Date: Tue, 8 Aug 2023 22:09:21 -0700 Subject: [PATCH 13/23] Remove confusing lock scope --- src/rogue/interfaces/memory/Emulate.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/rogue/interfaces/memory/Emulate.cpp b/src/rogue/interfaces/memory/Emulate.cpp index 838171e41..e873d2304 100644 --- a/src/rogue/interfaces/memory/Emulate.cpp +++ b/src/rogue/interfaces/memory/Emulate.cpp @@ -69,7 +69,7 @@ void rim::Emulate::doTransaction(rim::TransactionPtr tran) { // printf("Got transaction address=0x%" PRIx64 ", size=%" PRIu32 ", type = %" PRIu32 "\n", addr, size, type); - rogue::interfaces::memory::TransactionLockPtr lock = tran->lock(); + rogue::interfaces::memory::TransactionLockPtr tlock = tran->lock(); { std::lock_guard lock(mtx_); From 44757063fb8c9f727cceb68ffe049055a5ac7267 Mon Sep 17 00:00:00 2001 From: Larry Ruckman Date: Wed, 9 Aug 2023 10:41:46 -0700 Subject: [PATCH 14/23] enchancing test_epics.py adding remote variables to the test script --- tests/test_epics.py | 114 ++++++++++++++++++++++++++++++++++++-------- 1 file changed, 93 insertions(+), 21 deletions(-) diff --git a/tests/test_epics.py b/tests/test_epics.py index 6c2728b18..d3e166aee 100644 --- a/tests/test_epics.py +++ b/tests/test_epics.py @@ -9,32 +9,82 @@ # contained in the LICENSE.txt file. #----------------------------------------------------------------------------- -import pyrogue import time +import pyrogue as pr import pyrogue.protocols.epicsV4 +import rogue.interfaces.memory + from p4p.client.thread import Context epics_prefix='test_ioc' -class myDevice(pyrogue.Device): - def __init__(self, name="myDevice", description='My device', **kargs): - super().__init__(name=name, description=description, **kargs) +class SimpleDev(pr.Device): + + def __init__(self,**kwargs): + + super().__init__(**kwargs) + + self.add(pr.LocalVariable( + name = 'LocalRwInt', + value = 0, + mode = 'RW')) + + self.add(pr.LocalVariable( + name = 'LocalWoInt', + value = 0, + mode = 'WO')) + + self.add(pr.LocalVariable( + name = 'LocalRwFloat', + value = 0.0, + mode = 'RW')) + + self.add(pr.RemoteVariable( + name = "RemoteRwInt", + offset = 0x000, + value = 0, + mode = "RW", + )) - self.add(pyrogue.LocalVariable( - name='var', - value=0, - mode='RW')) + self.add(pr.RemoteVariable( + name = "RemoteWoInt", + offset = 0x004, + value = 0, + mode = "WO", + )) - self.add(pyrogue.LocalVariable( - name='var_float', - value=0.0, - mode='RW')) -class LocalRoot(pyrogue.Root): +class LocalRoot(pr.Root): def __init__(self): - pyrogue.Root.__init__(self, name='LocalRoot', description='Local root') - my_device=myDevice() - self.add(my_device) + pr.Root.__init__(self, name='LocalRoot', description='Local root') + + pr.Root.__init__(self, + name='LocalRoot', + description='Local root', + timeout=2.0, + pollEn=False) + + # Use a memory space emulator + sim = rogue.interfaces.memory.Emulate(4,0x1000) + self.addInterface(sim) + + # Create a memory gateway + ms = rogue.interfaces.memory.TcpServer("127.0.0.1",9080) + self.addInterface(ms) + + # Connect the memory gateways together + sim << ms + + # Create a memory gateway + mc = rogue.interfaces.memory.TcpClient("127.0.0.1",9080) + self.addInterface(mc) + + # Add Device + self.add(SimpleDev( + name = 'SimpleDev', + offset = 0x0000, + memBase = mc, + )) class LocalRootWithEpics(LocalRoot): def __init__(self, use_map=False): @@ -43,8 +93,11 @@ def __init__(self, use_map=False): if use_map: # PV map pv_map = { - 'LocalRoot.myDevice.var' : epics_prefix+':LocalRoot:myDevice:var', - 'LocalRoot.myDevice.var_float' : epics_prefix+':LocalRoot:myDevice:var_float', + 'LocalRoot.SimpleDev.LocalRwInt' : epics_prefix+':LocalRoot:SimpleDev:LocalRwInt', + 'LocalRoot.SimpleDev.LocalWoInt' : epics_prefix+':LocalRoot:SimpleDev:LocalWoInt', + 'LocalRoot.SimpleDev.LocalRwFloat' : epics_prefix+':LocalRoot:SimpleDev:LocalRwFloat', + 'LocalRoot.SimpleDev.RemoteRwInt' : epics_prefix+':LocalRoot:SimpleDev:RemoteRwInt', + 'LocalRoot.SimpleDev.RemoteWoInt' : epics_prefix+':LocalRoot:SimpleDev:RemoteWoInt', } else: pv_map=None @@ -75,7 +128,7 @@ def test_local_root(): time.sleep(1) # Device EPICS PV name prefix - device_epics_prefix=epics_prefix+':LocalRoot:myDevice' + device_epics_prefix=epics_prefix+':LocalRoot:SimpleDev' # Test dump method root.epics.dump() @@ -85,7 +138,7 @@ def test_local_root(): time.sleep(1) # Test RW a variable holding an scalar value - pv_name=device_epics_prefix+':var' + pv_name=device_epics_prefix+':LocalRwInt' test_value=314 ctxt.put(pv_name, test_value) time.sleep(1) @@ -93,8 +146,13 @@ def test_local_root(): if test_result != test_value: raise AssertionError('pv_name={}: test_value={}; test_result={}'.format(pv_name, test_value, test_result)) + # Test WO a variable holding an scalar value + pv_name=device_epics_prefix+':LocalWoInt' + test_value=314 + ctxt.put(pv_name, test_value) + # Test RW a variable holding a float value - pv_name=device_epics_prefix+':var_float' + pv_name=device_epics_prefix+':LocalRwFloat' test_value=5.67 ctxt.put(pv_name, test_value) time.sleep(1) @@ -102,6 +160,20 @@ def test_local_root(): if test_result != test_value: raise AssertionError('pvStates={} pv_name={}: test_value={}; test_result={}'.format(s, pv_name, test_value, test_result)) + # Test RW a variable holding an scalar value + pv_name=device_epics_prefix+':RemoteRwInt' + test_value=314 + ctxt.put(pv_name, test_value) + time.sleep(1) + test_result=ctxt.get(pv_name) + if test_result != test_value: + raise AssertionError('pv_name={}: test_value={}; test_result={}'.format(pv_name, test_value, test_result)) + + # Test WO a variable holding an scalar value + pv_name=device_epics_prefix+':RemoteWoInt' + test_value=314 + ctxt.put(pv_name, test_value) + # Allow epics client to reset time.sleep(5) From a40e8dca88660862bdb9f3a0d43a5c8d6e1e0707 Mon Sep 17 00:00:00 2001 From: Ryan Herbst Date: Wed, 9 Aug 2023 15:58:05 -0700 Subject: [PATCH 15/23] Add ensureSingleBuffer in slave class --- include/rogue/interfaces/stream/Slave.h | 14 ++++++++++++ src/rogue/interfaces/stream/Slave.cpp | 29 +++++++++++++++++++++++++ 2 files changed, 43 insertions(+) diff --git a/include/rogue/interfaces/stream/Slave.h b/include/rogue/interfaces/stream/Slave.h index c730e35ef..6af34787e 100644 --- a/include/rogue/interfaces/stream/Slave.h +++ b/include/rogue/interfaces/stream/Slave.h @@ -126,6 +126,20 @@ class Slave : public rogue::interfaces::stream::Pool, */ uint64_t getByteCount(); + //! Ensure frame is a single buffer + /** This method makes sure the passed frame is composed of a single buffer. + * If the reqNew flag is true and the passed frame is not a single buffer, a + * new frame will be requested and the frame data will be copied, with the passed + * frame pointer being updated. The return value will indicate if the frame is a + * single buffer at the end of the process. A frame lock must be held when this + * method is called. + * + * Not exposed to Python + * @param frame Reference to frame pointer (FramePtr) + * @param rewEn Flag to determine if a new frame should be requested + */ + bool ensureSingleBuffer(std::shared_ptr& frame, bool reqEn); + #ifndef NO_PYTHON //! Support << operator in python diff --git a/src/rogue/interfaces/stream/Slave.cpp b/src/rogue/interfaces/stream/Slave.cpp index 68fa6b323..3ea134b9e 100644 --- a/src/rogue/interfaces/stream/Slave.cpp +++ b/src/rogue/interfaces/stream/Slave.cpp @@ -140,6 +140,35 @@ uint64_t ris::Slave::getByteCount() { return (frameBytes_); } +// Ensure passed frame is a single buffer +bool ris::Slave::ensureSingleBuffer(ris::FramePtr& frame, bool reqEn) { + // Frame is a single buffer + if (frame->bufferCount() == 1) + return true; + + else if (!reqEn) + return false; + + else { + uint32_t size = frame->getPayload(); + ris::FramePtr nFrame = acceptReq(size, true); + + if (nFrame->bufferCount() != 1) + return false; + + else { + nFrame->setPayload(size); + + ris::FrameIterator srcIter = frame->begin(); + ris::FrameIterator dstIter = nFrame->begin(); + + ris::copyFrame(srcIter, size, dstIter); + frame = nFrame; + return true; + } + } +} + void ris::Slave::setup_python() { #ifndef NO_PYTHON From f7ee6e9a8084189e603353ce68d308865bda47d5 Mon Sep 17 00:00:00 2001 From: Ryan Herbst Date: Wed, 9 Aug 2023 23:47:39 -0700 Subject: [PATCH 16/23] Add local frame request --- include/rogue/interfaces/stream/Slave.h | 9 +++++++++ src/rogue/hardware/axi/AxiStreamDma.cpp | 2 +- src/rogue/interfaces/stream/Slave.cpp | 7 ++++++- src/rogue/interfaces/stream/TcpCore.cpp | 2 +- src/rogue/protocols/udp/Client.cpp | 4 ++-- src/rogue/protocols/udp/Server.cpp | 4 ++-- 6 files changed, 21 insertions(+), 7 deletions(-) diff --git a/include/rogue/interfaces/stream/Slave.h b/include/rogue/interfaces/stream/Slave.h index 6af34787e..029d44b1e 100644 --- a/include/rogue/interfaces/stream/Slave.h +++ b/include/rogue/interfaces/stream/Slave.h @@ -140,6 +140,15 @@ class Slave : public rogue::interfaces::stream::Pool, */ bool ensureSingleBuffer(std::shared_ptr& frame, bool reqEn); + // Process a local frame request + /* Method to service a local frame request + * + * size Minimum size for requested Frame, larger Frame may be allocated + * zeroCopyEn Flag which indicates if a zero copy mode Frame is allowed. + * Newly allocated Frame pointer (FramePtr) + */ + std::shared_ptr reqLocalFrame(uint32_t size, bool zeroCopyEn); + #ifndef NO_PYTHON //! Support << operator in python diff --git a/src/rogue/hardware/axi/AxiStreamDma.cpp b/src/rogue/hardware/axi/AxiStreamDma.cpp index 702aec446..d7330925a 100644 --- a/src/rogue/hardware/axi/AxiStreamDma.cpp +++ b/src/rogue/hardware/axi/AxiStreamDma.cpp @@ -270,7 +270,7 @@ ris::FramePtr rha::AxiStreamDma::acceptReq(uint32_t size, bool zeroCopyEn) { // Zero copy is disabled. Allocate from memory. if (zeroCopyEn == false || desc_->rawBuff == NULL) { - frame = ris::Pool::acceptReq(size, false); + frame = reqLocalFrame(size, false); } // Allocate zero copy buffers from driver diff --git a/src/rogue/interfaces/stream/Slave.cpp b/src/rogue/interfaces/stream/Slave.cpp index 3ea134b9e..3ba5107c7 100644 --- a/src/rogue/interfaces/stream/Slave.cpp +++ b/src/rogue/interfaces/stream/Slave.cpp @@ -151,7 +151,7 @@ bool ris::Slave::ensureSingleBuffer(ris::FramePtr& frame, bool reqEn) { else { uint32_t size = frame->getPayload(); - ris::FramePtr nFrame = acceptReq(size, true); + ris::FramePtr nFrame = reqLocalFrame(size, true); if (nFrame->bufferCount() != 1) return false; @@ -169,6 +169,11 @@ bool ris::Slave::ensureSingleBuffer(ris::FramePtr& frame, bool reqEn) { } } +// Process a local frame request +ris::FramePtr ris::Slave::reqLocalFrame(uint32_t size, bool zeroCopyEn) { + return ris::Pool::acceptReq(size,zeroCopyEn); +} + void ris::Slave::setup_python() { #ifndef NO_PYTHON diff --git a/src/rogue/interfaces/stream/TcpCore.cpp b/src/rogue/interfaces/stream/TcpCore.cpp index 00523de97..2a5d91b4a 100644 --- a/src/rogue/interfaces/stream/TcpCore.cpp +++ b/src/rogue/interfaces/stream/TcpCore.cpp @@ -271,7 +271,7 @@ void ris::TcpCore::runThread() { size = zmq_msg_size(&(msg[3])); // Generate frame - frame = ris::Pool::acceptReq(size, false); + frame = reqLocalFrame(size, false); frame->setPayload(size); // Copy data diff --git a/src/rogue/protocols/udp/Client.cpp b/src/rogue/protocols/udp/Client.cpp index d8ad96dd3..579079d8a 100644 --- a/src/rogue/protocols/udp/Client.cpp +++ b/src/rogue/protocols/udp/Client.cpp @@ -197,7 +197,7 @@ void rpu::Client::runThread(std::weak_ptr lockPtr) { udpLog_->logThreadId(); // Preallocate frame - frame = ris::Pool::acceptReq(maxPayload(), false); + frame = reqLocalFrame(maxPayload(), false); while (threadEn_) { // Attempt receive @@ -215,7 +215,7 @@ void rpu::Client::runThread(std::weak_ptr lockPtr) { } // Get new frame - frame = ris::Pool::acceptReq(maxPayload(), false); + frame = reqLocalFrame(maxPayload(), false); } else { // Setup fds for select call FD_ZERO(&fds); diff --git a/src/rogue/protocols/udp/Server.cpp b/src/rogue/protocols/udp/Server.cpp index 7aa0826ee..03fec128a 100644 --- a/src/rogue/protocols/udp/Server.cpp +++ b/src/rogue/protocols/udp/Server.cpp @@ -200,7 +200,7 @@ void rpu::Server::runThread(std::weak_ptr lockPtr) { udpLog_->logThreadId(); // Preallocate frame - frame = ris::Pool::acceptReq(maxPayload(), false); + frame = reqLocalFrame(maxPayload(), false); while (threadEn_) { // Attempt receive @@ -219,7 +219,7 @@ void rpu::Server::runThread(std::weak_ptr lockPtr) { } // Get new frame - frame = ris::Pool::acceptReq(maxPayload(), false); + frame = reqLocalFrame(maxPayload(), false); // Lock before updating address if (memcmp(&remAddr_, &tmpAddr, sizeof(remAddr_)) != 0) { From 675368977348595ef7233f42841f52638afc9c26 Mon Sep 17 00:00:00 2001 From: Ryan Herbst Date: Thu, 10 Aug 2023 10:26:48 -0700 Subject: [PATCH 17/23] Fix Anaconda documentation --- docs/src/installing/anaconda.rst | 5 ++- docs/src/installing/anaconda_build.rst | 55 +++++++------------------- docs/src/installing/build.rst | 21 +--------- 3 files changed, 19 insertions(+), 62 deletions(-) diff --git a/docs/src/installing/anaconda.rst b/docs/src/installing/anaconda.rst index b9b0efe66..f0c77f4a2 100644 --- a/docs/src/installing/anaconda.rst +++ b/docs/src/installing/anaconda.rst @@ -17,8 +17,8 @@ Go to https://www.anaconda.com/download to get the latest version of anaconda. E .. code:: - $ wget https://repo.anaconda.com/archive/Anaconda3-2023.03-1-Linux-x86_64.sh - $ bash Anaconda3-2023.03-1-Linux-x86_64.sh + $ wget https://repo.anaconda.com/archive/Anaconda3-{version}-Linux-x86_64.sh + $ bash Anaconda3-{version}-Linux-x86_64.sh You do not need to install visual studio. @@ -32,6 +32,7 @@ It is important to use the latest conda solver: .. code:: + $ conda activate $ conda config --set channel_priority strict $ conda install -n base conda-libmamba-solver $ conda config --set solver libmamba diff --git a/docs/src/installing/anaconda_build.rst b/docs/src/installing/anaconda_build.rst index 5ddd72c4b..d8b28e5f2 100644 --- a/docs/src/installing/anaconda_build.rst +++ b/docs/src/installing/anaconda_build.rst @@ -5,25 +5,21 @@ Building Rogue Inside Anaconda ============================== This section provides instructions for downloading and building rogue inside an anaconda environment. These -instructions are relevant for Linux, Ubuntu on Windows and MacOS. - -See MacOS section at the bottom for additional steps required for building rogue in MacOS. +instructions are relevant for Linux See the section :ref:`installing_windows` for additional steps required for Windows. Getting Anaconda ================ -Download and install anaconda (or miniconda) if you don't already have it installed on your machine. Choose an install location with a lot of available diskspace (> 5GB). Anaconda appears to only work reliably in the bash shell. +Download and install anaconda (or miniconda) if you don't already have it installed on your machine. Choose an install location with a lot of available diskspace (> 5GB). Anaconda appears to only work reliably in the bash shell. Go to https://www.anaconda.com/download to get the latest version of anaconda. Example steps for installing anaconda are included below: .. code:: - $ wget https://repo.anaconda.com/archive/Anaconda3-5.3.0-Linux-x86_64.sh - $ bash Anaconda3-5.3.0-Linux-x86_64.sh - -You do not need to install visual studio. + $ wget https://repo.anaconda.com/archive/Anaconda3-{version}-Linux-x86_64.sh + $ bash Anaconda3-{version}-Linux-x86_64.sh Use the following command to add anaconda to your environment. This can be added to your .bash_profile. @@ -31,6 +27,15 @@ Use the following command to add anaconda to your environment. This can be added $ source /path/to/my/anaconda3/etc/profile.d/conda.sh +Set your local anaconda to use the update solver: + +.. code:: + + $ conda activate + $ conda config --set channel_priority strict + $ conda install -n base conda-libmamba-solver + $ conda config --set solver libmamba + Downloading Rogue & Creating Anaconda Environment ================================================= @@ -38,24 +43,11 @@ The next step is to download rogue and create a rogue compatible anaconda enviro .. code:: - $ conda activate - $ conda install git $ git clone https://github.com/slaclab/rogue.git $ cd rogue - -For Linux: - -.. code:: - + $ conda activate $ conda env create -n rogue_build -f conda.yml -For MacOS: - -.. code:: - - $ conda env create -n rogue_build -f conda_mac.yml - - You now have an anaconda environment named rogue_build which contains all of the packages required to build and run rogue. To activate this environment: @@ -77,7 +69,7 @@ Once the rogue environment is activated, you can build and install rogue $ make $ make install -The Rogue build system will automatically detect that it is in a conda environment and it will be installed +The Rogue build system will automatically detect that it is in a conda environment and it will be installed within the anaconda rogue environment. Using Rogue In Anaconda @@ -122,20 +114,3 @@ Run the following commands to delete the anaconda environment. $ conda env remove -n rogue_build -Special Steps For MacOS -======================= - -In order to compile rogue in MacOS you first need to download an install an older version of the MacOS SDK - -.. code:: - - $ git clone https://github.com/phracker/MacOSX-SDKs - $ sudo mv MacOSX-SDKs/MacOSX10.9.sdk /opt/ - -You must set the following environment variables to setup anaconda in build mode before creating and activating the rogue environment. - -.. code:: - - $ export CONDA_BUILD_SYSROOT=/opt/MacOSX10.9.sdk - $ export CONDA_BUILD=1 - diff --git a/docs/src/installing/build.rst b/docs/src/installing/build.rst index 18ea13f89..414826115 100644 --- a/docs/src/installing/build.rst +++ b/docs/src/installing/build.rst @@ -5,7 +5,7 @@ Building Rogue From Source ========================== The following instructions demonstrate how to build rogue outside of the anaconda environment. These -instructions are only relevant for the Linux and MacOS operating systems. See :ref:`installing_anaconda` or +instructions are only relevant for the Linux operating systems. See :ref:`installing_anaconda` or :ref:`installing_docker` for Windows and MacOS. Installing Packages Required For Rogue @@ -50,25 +50,6 @@ archlinux: $ pacman -S zeromq $ pacman -S python-pyqt5 -MacOs: -####### - -Information on the homebrew package manager can be found at: ``_ - -.. code:: - - $ brew install cmake - $ brew install python3 - $ brew install boost - $ brew install bzip2 - $ brew install python-pip - $ brew install git - $ brew install zeromq - $ brew install pyqt5 - -Epics V3 support is and optional module that will be included in the rogue build -if the EPICS_BASE directory is set in the user's environment. - Building & Installing Rogue =========================== From 8e915d2d187718ce63b965161d5c5f552ef2f20a Mon Sep 17 00:00:00 2001 From: Ryan Herbst Date: Thu, 10 Aug 2023 10:29:31 -0700 Subject: [PATCH 18/23] Fix docker reference --- docs/src/installing/build.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/src/installing/build.rst b/docs/src/installing/build.rst index 414826115..c7d3db6e7 100644 --- a/docs/src/installing/build.rst +++ b/docs/src/installing/build.rst @@ -5,7 +5,7 @@ Building Rogue From Source ========================== The following instructions demonstrate how to build rogue outside of the anaconda environment. These -instructions are only relevant for the Linux operating systems. See :ref:`installing_anaconda` or +instructions are only relevant for the Linux operating systems. See :ref:`installing_docker` for Windows and MacOS. Installing Packages Required For Rogue From d1ae5155d1de2cdf7479dd7d0f145aa081fed3f0 Mon Sep 17 00:00:00 2001 From: Ryan Herbst Date: Thu, 10 Aug 2023 10:30:45 -0700 Subject: [PATCH 19/23] Remove windows --- docs/src/installing/anaconda_build.rst | 2 -- 1 file changed, 2 deletions(-) diff --git a/docs/src/installing/anaconda_build.rst b/docs/src/installing/anaconda_build.rst index d8b28e5f2..037a90416 100644 --- a/docs/src/installing/anaconda_build.rst +++ b/docs/src/installing/anaconda_build.rst @@ -7,8 +7,6 @@ Building Rogue Inside Anaconda This section provides instructions for downloading and building rogue inside an anaconda environment. These instructions are relevant for Linux -See the section :ref:`installing_windows` for additional steps required for Windows. - Getting Anaconda ================ From d7dcd33eeca7ba0ba2279625460547a6ccba6bff Mon Sep 17 00:00:00 2001 From: Ryan Herbst Date: Tue, 22 Aug 2023 09:26:32 -0700 Subject: [PATCH 20/23] Performance Fixes For Process --- python/pyrogue/_Process.py | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/python/pyrogue/_Process.py b/python/pyrogue/_Process.py index 415da9cf8..c38e7291f 100644 --- a/python/pyrogue/_Process.py +++ b/python/pyrogue/_Process.py @@ -67,8 +67,9 @@ def __init__(self, *, argVariable=None, returnVariable=None, function=None, **kw self.add(pr.LocalVariable( name = 'Step', mode = 'RO', + pollInterval=1.0, value = 1, - description = "Total number of loops steps for the process")) + description = "Current number of steps")) self.add(pr.LocalVariable( name = 'TotalSteps', @@ -77,9 +78,8 @@ def __init__(self, *, argVariable=None, returnVariable=None, function=None, **kw description = "Total number of loops steps for the process")) @self.command(hidden=True) - def Advance(): - self.Step += 1 - self.Progress.set(self.Step.value()/self.TotalSteps.value()) + def Advance(arg): + self._incrementSteps(1,write=True) # Add arg variable if not already added if self._argVar is not None and self._argVar not in self: @@ -89,6 +89,14 @@ def Advance(): if self._retVar is not None and self._retVar not in self: self.add(self._retVar) + def _incrementSteps(self, incr, write=False): + with self.Step.lock: + self.Step.set(self.Step.value() + incr,write=write) + self.Progress.set(self.Step.value()/self.TotalSteps.value(),write=write) + + def _setSteps(self, value, write=False): + self.Step.set(value,write=write) + self.Progress.set(self.Step.value()/self.TotalSteps.value(),write=write) def _startProcess(self): """ """ From 693e7a343b5cff6730fea22d6c8f46d9afc41a3b Mon Sep 17 00:00:00 2001 From: Ryan Herbst Date: Tue, 22 Aug 2023 09:28:14 -0700 Subject: [PATCH 21/23] Fix --- python/pyrogue/_Process.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/python/pyrogue/_Process.py b/python/pyrogue/_Process.py index c38e7291f..5f8ec7982 100644 --- a/python/pyrogue/_Process.py +++ b/python/pyrogue/_Process.py @@ -79,7 +79,7 @@ def __init__(self, *, argVariable=None, returnVariable=None, function=None, **kw @self.command(hidden=True) def Advance(arg): - self._incrementSteps(1,write=True) + self._incrementSteps(1) # Add arg variable if not already added if self._argVar is not None and self._argVar not in self: @@ -89,14 +89,14 @@ def Advance(arg): if self._retVar is not None and self._retVar not in self: self.add(self._retVar) - def _incrementSteps(self, incr, write=False): + def _incrementSteps(self, incr): with self.Step.lock: - self.Step.set(self.Step.value() + incr,write=write) - self.Progress.set(self.Step.value()/self.TotalSteps.value(),write=write) + self.Step.set(self.Step.value() + incr,write=False) + self.Progress.set(self.Step.value()/self.TotalSteps.value(),write=False) - def _setSteps(self, value, write=False): - self.Step.set(value,write=write) - self.Progress.set(self.Step.value()/self.TotalSteps.value(),write=write) + def _setSteps(self, value): + self.Step.set(value,write=False) + self.Progress.set(self.Step.value()/self.TotalSteps.value(),write=False) def _startProcess(self): """ """ From 05b9b567ab2c0b86f7af659dd67365679aa5ddc3 Mon Sep 17 00:00:00 2001 From: Ryan Herbst Date: Tue, 22 Aug 2023 09:39:05 -0700 Subject: [PATCH 22/23] Override bitsize for list variables, make offset mandatory --- python/pyrogue/_Variable.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/python/pyrogue/_Variable.py b/python/pyrogue/_Variable.py index 973aaeb72..78d166a82 100644 --- a/python/pyrogue/_Variable.py +++ b/python/pyrogue/_Variable.py @@ -954,7 +954,7 @@ def __init__(self, *, highWarning=None, highAlarm=None, base=pr.UInt, - offset=None, + offset, numValues=0, valueBits=0, valueStride=0, @@ -1017,7 +1017,8 @@ def __init__(self, *, pollInterval=pollInterval,updateNotify=updateNotify, guiGroup=guiGroup, **kwargs) - # If numValues > 0 the bit size array must only have one entry and the total number of bits must be a multiple of the number of values + # If numValues > 0 the bit size array must only have one entry + # Auto calculate the total number of bits if numValues != 0: self._nativeType = np.ndarray self._ndType = self._base.ndType @@ -1029,8 +1030,8 @@ def __init__(self, *, if valueBits > valueStride: raise VariableError(f'ValueBits {valueBits} is greater than valueStrude {valueStride}') - if (numValues * valueStride) != sum(bitSize): - raise VariableError(f'Total bitSize {sum(bitSize)} is not equal to multile of numValues {numValues} and valueStride {valueStride}') + # Override the bitSize + bitSize = [numValues * valueStride] if self._ndType is None: raise VariableError(f'Invalid base type {self._base} with numValues = {numValues}') From 2f2f7241bd7d2703d7771299c03d72fa4bf1ff02 Mon Sep 17 00:00:00 2001 From: Ryan Herbst Date: Tue, 22 Aug 2023 09:41:08 -0700 Subject: [PATCH 23/23] Fix" --- python/pyrogue/_Variable.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/pyrogue/_Variable.py b/python/pyrogue/_Variable.py index 78d166a82..26983dde4 100644 --- a/python/pyrogue/_Variable.py +++ b/python/pyrogue/_Variable.py @@ -1031,7 +1031,7 @@ def __init__(self, *, raise VariableError(f'ValueBits {valueBits} is greater than valueStrude {valueStride}') # Override the bitSize - bitSize = [numValues * valueStride] + bitSize[0] = numValues * valueStride if self._ndType is None: raise VariableError(f'Invalid base type {self._base} with numValues = {numValues}')