add initial stress tests

.github/workflows/stress.yml

@@ -0,0 +1,73 @@
+# Copyright (c) 2020, The OTNS Authors.
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are met:
+# 1. Redistributions of source code must retain the above copyright
+#    notice, this list of conditions and the following disclaimer.
+# 2. Redistributions in binary form must reproduce the above copyright
+#    notice, this list of conditions and the following disclaimer in the
+#    documentation and/or other materials provided with the distribution.
+# 3. Neither the name of the copyright holder nor the
+#    names of its contributors may be used to endorse or promote products
+#    derived from this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
+# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+# POSSIBILITY OF SUCH DAMAGE.
+
+name: Stress
+
+on: [pull_request]
+
+jobs:
+  cancel-previous-runs:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: rokroskar/workflow-run-cleanup-action@master
+        env:
+          GITHUB_TOKEN: "${{ secrets.GITHUB_TOKEN }}"
+        if: "github.ref != 'refs/heads/master'"
+
+  stress-tests:
+    name: Stress Tests
+    strategy:
+      matrix:
+        python-version: [3.8]
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/setup-go@v1
+        with:
+          go-version: '1.13'
+      - name: Set up Python ${{ matrix.python-version }}
+        uses: actions/setup-python@v1
+        with:
+          python-version: ${{ matrix.python-version }}
+      - run: |
+          mkdir -p /home/runner/work/_temp/_github_home
+      - uses: actions/checkout@v2
+      - name: Stress Test
+        env:
+          STRESS_RESULT_FILE: /home/runner/work/_temp/_github_home/stress.log
+        run: |
+          ./script/test stress-tests
+      - run: |
+          echo "Stress Test Results:"
+          cat /home/runner/work/_temp/_github_home/stress.log
+      - uses: simonlingoogle/comment-on-pr@master
+        name: Comment With Stress Result
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        with:
+          filename: /github/home/stress.log
+          duplicate_check: "[OTNS](https://github.com/openthread/ot-ns) Stress Tests Report"
+      - run: |
+          test -f /tmp/run-stress-suite-ok

cli/CmdRunner.go

@@ -443,8 +443,8 @@ func (rt *CmdRunner) executeLsNodes(cc *CommandContext, cmd *NodesCmd) {
 		for nodeid := range sim.Nodes() {
 			dnode := sim.Dispatcher().GetNode(nodeid)
 			var line strings.Builder
-			line.WriteString(fmt.Sprintf("id=%d\textaddr=%016x\trloc16=%04x\tx=%d\ty=%d\tfailed=%v", nodeid, dnode.ExtAddr, dnode.Rloc16,
-				dnode.X, dnode.Y, dnode.IsFailed()))
+			line.WriteString(fmt.Sprintf("id=%d\textaddr=%016x\trloc16=%04x\tx=%d\ty=%d\tstate=%s\tfailed=%v", nodeid, dnode.ExtAddr, dnode.Rloc16,
+				dnode.X, dnode.Y, dnode.Role, dnode.IsFailed()))
 			cc.outputf("%s\n", line.String())
 		}
 	})

dispatcher/Node.go

@@ -71,6 +71,7 @@ type Node struct {
 	PartitionId uint32
 	ExtAddr     uint64
 	Rloc16      uint16
+	Role        OtDeviceRole
 	CreateTime  uint64
 	CurTime     uint64
 
@@ -99,6 +100,7 @@ func newNode(d *Dispatcher, nodeid NodeId, extaddr uint64, x, y int, radioRange
 		Y:           y,
 		ExtAddr:     extaddr,
 		Rloc16:      threadconst.InvalidRloc16,
+		Role:        OtDeviceRoleDisabled,
 		addr:        addr,
 		radioRange:  radioRange,
 		joinerState: OtJoinerStateIdle,

dispatcher/dispatcher.go

@@ -715,7 +715,7 @@ func (d *Dispatcher) handleStatusPush(srcid NodeId, data string) {
 		if sp[0] == "role" {
 			role, err := strconv.Atoi(sp[1])
 			simplelogger.PanicIfError(err)
-			d.vis.SetNodeRole(srcid, visualize.OtDeviceRole(role))
+			d.setNodeRole(srcid, OtDeviceRole(role))
 		} else if sp[0] == "rloc16" {
 			rloc16, err := strconv.Atoi(sp[1])
 			simplelogger.PanicIfError(err)
@@ -993,3 +993,14 @@ func (d *Dispatcher) onStatusPushExtAddr(node *Node, oldExtAddr uint64) {
 	d.extaddrMap[node.ExtAddr] = node
 	d.vis.OnExtAddrChange(node.Id, node.ExtAddr)
 }
+
+func (d *Dispatcher) setNodeRole(id NodeId, role OtDeviceRole) {
+	node := d.nodes[id]
+	if node == nil {
+		simplelogger.Warnf("setNodeRole: node %d not found", id)
+		return
+	}
+
+	node.Role = role
+	d.vis.SetNodeRole(id, role)
+}

pylibs/otns/cli/OTNS.py

@@ -110,6 +110,14 @@ def speed(self, speed: float):
 
         self._do_command(f'speed {speed}')
 
+    def set_poll_period(self, nodeid: int, period: float) -> None:
+        ms = int(period * 1000)
+        self.node_cmd(nodeid, f'pollperiod {ms}')
+
+    def get_poll_period(self, nodeid: int) -> float:
+        ms = self._expect_int(self.node_cmd(nodeid, 'pollperiod'))
+        return ms / 1000.0
+
     def _detect_otns_path(self) -> str:
         env_otns_path = os.getenv('OTNS')
         if env_otns_path:
@@ -401,6 +409,7 @@ def node_cmd(self, nodeid: int, cmd: str) -> List[str]:
         :param cmd: command to execute
         :return: lines of command output
         """
+        assert nodeid >= 0, f'invalid node ID: {nodeid}'
         cmd = f'node {nodeid} "{cmd}"'
         output = self._do_command(cmd)
         return output
@@ -550,6 +559,42 @@ def commissioner_joiner_add(self, nodeid: int, usr: str, pwd: str, timeout=None)
         timeout_s = f" {timeout}" if timeout is not None else ""
         self.node_cmd(nodeid, f"commissioner joiner add {usr} {pwd}{timeout_s}")
 
+    def get_router_upgrade_threshold(self, nodeid: int) -> int:
+        """
+        Get Router upgrade threshold.
+
+        :param nodeid: the node ID
+        :return: the Router upgrade threshold
+        """
+        return self._expect_int(self.node_cmd(nodeid, 'routerupgradethreshold'))
+
+    def set_router_upgrade_threshold(self, nodeid: int, val: int) -> None:
+        """
+        Set Router upgrade threshold.
+
+        :param nodeid: the node ID
+        :param val: the Router upgrade threshold
+        """
+        self.node_cmd(nodeid, f'routerupgradethreshold {val}')
+
+    def get_router_downgrade_threshold(self, nodeid: int) -> int:
+        """
+        Get Router downgrade threshold.
+
+        :param nodeid: the node ID
+        :return: the Router downgrade threshold
+        """
+        return self._expect_int(self.node_cmd(nodeid, 'routerdowngradethreshold'))
+
+    def set_router_downgrade_threshold(self, nodeid: int, val: int) -> None:
+        """
+        Set Router downgrade threshold.
+
+        :param nodeid: the node ID
+        :param val: the Router downgrade threshold
+        """
+        self.node_cmd(nodeid, f'routerdowngradethreshold {val}')
+
     def _expect_int(self, output: List[str]) -> int:
         assert len(output) == 1, output
         return int(output[0])

pylibs/stress_tests/BaseStressTest.py

@@ -0,0 +1,163 @@
+#!/usr/bin/env python3
+# Copyright (c) 2020, The OTNS Authors.
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are met:
+# 1. Redistributions of source code must retain the above copyright
+#    notice, this list of conditions and the following disclaimer.
+# 2. Redistributions in binary form must reproduce the above copyright
+#    notice, this list of conditions and the following disclaimer in the
+#    documentation and/or other materials provided with the distribution.
+# 3. Neither the name of the copyright holder nor the
+#    names of its contributors may be used to endorse or promote products
+#    derived from this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
+# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+# POSSIBILITY OF SUCH DAMAGE.
+import ipaddress
+import logging
+import os
+import sys
+import time
+import traceback
+from functools import wraps
+from typing import Collection
+
+from StressTestResult import StressTestResult
+from errors import UnexpectedNodeState, UnexpectedError, UnexpectedNodeAddr
+from otns.cli import OTNS
+
+
+class StressTestMetaclass(type):
+    def __new__(cls, name, bases, dct):
+        assert 'run' in dct, f'run method is not defined in {name}'
+
+        orig_run = dct.pop('run')
+
+        @wraps(orig_run)
+        def run_wrapper(self: 'BaseStressTest', report=True):
+            try:
+                orig_run(self)
+            except Exception as ex:
+                traceback.print_exc()
+                self.result.fail_with_error(ex)
+            finally:
+                self.stop()
+
+            if report:
+                self.report()
+
+        dct['run'] = run_wrapper
+
+        t = super().__new__(cls, name, bases, dct)
+        return t
+
+
+class BaseStressTest(object, metaclass=StressTestMetaclass):
+    def __init__(self, name, headers, raw=False):
+        DEBUG = int(os.getenv('DEBUG', '0'))
+        logging.basicConfig(level=logging.DEBUG if DEBUG else logging.WARN)
+
+        self.name = name
+        self._otns_args = []
+        if raw:
+            self._otns_args.append('-raw')
+        self.ns = OTNS(otns_args=self._otns_args)
+        self.ns.speed = float('inf')
+        self.ns.web()
+
+        self.result = StressTestResult(name=name, headers=headers)
+        self.result.start()
+
+    def run(self):
+        raise NotImplementedError()
+
+    def reset(self):
+        nodes = self.ns.nodes()
+        if nodes:
+            self.ns.delete(*nodes.keys())
+
+    def stop(self):
+        self.result.stop()
+        self.ns.close()
+
+    def report(self):
+        try:
+            STRESS_RESULT_FILE = os.environ['STRESS_RESULT_FILE']
+            stress_result_fd = open(STRESS_RESULT_FILE, 'wt')
+        except KeyError:
+            stress_result_fd = sys.stdout
+
+        with stress_result_fd:
+            stress_result_fd.write(
+                f"""**[OTNS](https://github.com/openthread/ot-ns) Stress Tests Report Generated at {time.strftime(
+                    "%m/%d %H:%M:%S")}**\n""")
+            stress_result_fd.write(self.result.format())
+
+    def expect_node_state(self, nid: int, state: str, timeout: float, go_step: int = 1) -> None:
+        while timeout > 0:
+            self.ns.go(go_step)
+            timeout -= go_step
+
+            if self.ns.get_state(nid) == state:
+                return
+
+        raise UnexpectedNodeState(nid, state, self.ns.get_state(nid))
+
+    def expect_all_nodes_become_routers(self, timeout: int = 1000) -> None:
+        all_routers = False
+
+        while timeout > 0 and not all_routers:
+            self.ns.go(10)
+            timeout -= 10
+
+            nodes = (self.ns.nodes())
+
+            all_routers = True
+            for nid, info in nodes.items():
+                if info['state'] not in ['leader', 'router']:
+                    all_routers = False
+                    break
+
+            if all_routers:
+                break
+
+        if not all_routers:
+            raise UnexpectedError("not all nodes are Routers: %s" % self.ns.nodes())
+
+    def expect_node_addr(self, nodeid: int, addr: str, timeout=100):
+        addr = ipaddress.IPv6Address(addr)
+
+        found_addr = False
+        while timeout > 0:
+            if addr in map(ipaddress.IPv6Address, self.ns.get_ipaddrs(nodeid)):
+                found_addr = True
+                break
+
+            self.ns.go(1)
+
+        if not found_addr:
+            raise UnexpectedNodeAddr(f'Address {addr} not found on node {nodeid}')
+
+    def avg(self, vals: Collection[float]) -> float:
+        assert len(vals) > 0
+        return sum(vals) / len(vals)
+
+    def avg_except_max(self, vals: Collection[float]) -> float:
+        assert len(vals) >= 2
+        max_val = max(vals)
+        max_idxes = [i for i in range(len(vals)) if vals[i] >= max_val]
+        assert max_idxes
+        rmidx = max_idxes[0]
+        vals[rmidx:rmidx + 1] = []
+        return self.avg(vals)