multi_procnetdev2_test

#!/usr/bin/env python3

#
# samp1 - multi-instance procnetdev2
# samp2 - traditional procnetdev2 single load + config
#

import os
import re
import pwd
import sys
import json
import time
import argparse
import TADA
import logging
import atexit

from distutils.spawn import find_executable
from LDMS_Test import LDMSDCluster, LDMSDContainer, process_args, \
                      add_common_args, jprint, parse_ldms_ls, py_pty

if __name__ != "__main__":
    raise RuntimeError("This should not be impoarted as a module.")

class Debug(object):
    pass
D = Debug()

logging.basicConfig(format = "%(asctime)s %(name)s %(levelname)s %(message)s",
                    level = logging.INFO)

log = logging.getLogger(__name__)

exec(open(os.getenv("PYTHONSTARTUP", "/dev/null")).read())

#### default values #### ---------------------------------------------
sbin_ldmsd = find_executable("ldmsd")
if sbin_ldmsd:
    default_prefix, a, b = sbin_ldmsd.rsplit('/', 2)
else:
    default_prefix = "/opt/ovis"

#### argument parsing #### -------------------------------------------
ap = argparse.ArgumentParser(description = "Test multiple procnetdev2 instances" )
add_common_args(ap)
args = ap.parse_args()
process_args(args)

#### config variables #### ------------------------------
USER = args.user
PREFIX = args.prefix
COMMIT_ID = args.commit_id
SRC = args.src
CLUSTERNAME = args.clustername
DB = args.data_root
LDMSD_PORT = 411

#### spec #### -------------------------------------------------------
spec = {
    "name" : CLUSTERNAME,
    "description" : "{}'s test_agg cluster".format(USER),
    "type" : "NA",
    "templates" : { # generic template can apply to any object by "!extends"
        "ldmsd-base" : {
            "type" : "ldmsd",
            "listen" : [
                { "port" : LDMSD_PORT, "xprt" : "sock" },
            ],
        },
        "prdcr" : {
            "host" : "%name%",
            "port" : LDMSD_PORT,
            "xprt" : "sock",
            "type" : "active",
            "interval" : 1000000,
        },
    }, # templates
    "nodes" : [
        {
            "hostname" : "node-1",
            "component_id" : 1,
            "daemons" : [
                {
                    "name" : "sshd", # for debugging
                    "type" : "sshd",
                },
                {
                    "name" : "sampler-daemon",
                    "!extends" : "ldmsd-base",
                    "config" : [
                        f"load name=p0 plugin=procnetdev2",
                        f"config name=p0 producer=%hostname%"
                        f"       instance=%hostname%/netdev0"
                        f"       ifaces=eth0,eth1",
                        f"start name=p0 interval=1s offset=0",

                        f"load name=p1 plugin=procnetdev2",
                        f"config name=p1 producer=%hostname%"
                        f"       instance=%hostname%/netdev1"
                        f"       excludes=eth0,eth1",
                        f"start name=p1 interval=1s offset=0",
                    ],
                },
            ]
        },
        {
            "hostname" : "node-2",
            "component_id" : 2,
            "daemons" : [
                {
                    "name" : "sshd", # for debugging
                    "type" : "sshd",
                },
                {
                    "name" : "sampler-daemon",
                    "!extends" : "ldmsd-base",
                    "config" : [
                        f"load name=procnetdev2 plugin=procnetdev2",
                        f"config name=procnetdev2 producer=%hostname%"
                        f"       instance=%hostname%/netdev2",
                        f"start name=procnetdev2 interval=1s offset=0",
                    ],
                },
            ]
        },
        {
            "hostname" : "agg-1",
            "daemons" : [
                {
                    "name" : "sshd",
                    "type" : "sshd",
                },
                {
                    "name" : "agg",
                    "!extends" : "ldmsd-base",
                    "offset" : 200000,
                    "prdcrs" : [ # these producers will turn into `prdcr_add`
                        {
                            "name" : "node-1",
                            "!extends" : "prdcr",
                        },
                        {
                            "name" : "node-2",
                            "!extends" : "prdcr",
                        },
                    ],
                    "config" : [
                        f"prdcr_start_regex regex=.*",
                        f"updtr_add name=all interval=1000000 offset=200000",
                        f"updtr_prdcr_add name=all regex=.*",
                        f"updtr_start name=all",
                    ],
                },
            ]
        }
    ], # nodes

    #"image": "ovis-centos-build:slurm",
    "cap_add": [ "SYS_PTRACE", "SYS_ADMIN", "NET_ADMIN" ],
    "image": args.image,
    "ovis_prefix": PREFIX,
    "env" : { "FOO": "BAR" },
    "mounts": [
        f"{DB}:/db:rw",
        f"{os.path.realpath(sys.path[0])}:/tada-src:ro",
    ] + args.mount +
    ( ["{0}:{0}:ro".format(SRC)] if SRC else [] ),
}

#### test definition ####

test = TADA.Test(test_suite = "LDMSD",
                 test_type = "FVT",
                 test_name = "multi_procnetdev2_test",
                 test_desc = "Test multi-instance procnetdev2",
                 test_user = args.user,
                 commit_id = COMMIT_ID,
                 tada_addr = args.tada_addr)
test.add_assertion(1, "Load multiple procnetdev2s.")
test.add_assertion(2, "Load procnetdev2 (old style).")
test.add_assertion(3, "Sets being created accordingly.")
test.add_assertion(4, "Sets being updated accordingly (new network interface).")
test.add_assertion(5, "Sets being updated accordingly (network interface removal).")
test.add_assertion(6, "The set is removed from node-1 when procnetdev2 terminated.")
test.add_assertion(7, "The set is removed from agg-1 when procnetdev2 terminated.")

#### Start! ####
cluster = None
test.start()

@atexit.register
def at_exit():
    rc = test.finish()
    if cluster is not None:
        cluster.remove()
    os._exit(rc)

log.info("-- Get or create the cluster --")
cluster = LDMSDCluster.get(spec["name"], create = True, spec = spec)

agg1 = cluster.get_container("agg-1")
node1 = cluster.get_container("node-1")
node2 = cluster.get_container("node-2")

log.info("-- Start daemons --")
cluster.start_daemons()
cluster.make_known_hosts()

log.info("... wait a bit to make sure ldmsd's are up and data are flowing in")
time.sleep(10)

def verify_data(test_id, data, cmp):
    if set(data.keys()) != set(cmp.keys()):
        test.assert_test(test_id, False,
                    f"Expecting '{cmp.keys()}', but got '{data.keys()}'")
        return False
    for k in data.keys():
        d = data[k]['data']['netdev_list']
        c = cmp[k]['data']['netdev_list']
        d_names = [ o['name'] for o in d ]
        c_names = [ o['name'] for o in c ]

        d_names.sort()
        c_names.sort()

        if d_names != c_names:
            test.assert_test(test_id, False,
                         f"Expecting netdev names {c_names}, but got {d_names}")
            return False
    test.assert_test(test_id, True, f"verified")
    return True


# test.add_assertion(1, "Load multiple procnetdev2s.")
cond1 = False
while True: # will break
    rc1, out1 = agg1.exec_run("ldmsd_controller -x sock -p 411 -h node-1 <<<plugn_status")

    if rc1 != 0:
        test.assert_test(1, False, f"plugn_status error, rc: {rc1}, out: {out1}")
        break
    l11, l12 = out1.splitlines()[2:]
    if not l11.startswith('p0 ') or not l12.startswith('p1 '):
        test.assert_test(1, False, f"Bad output: {out1}")
        break
    test.assert_test(1, True, "verified")
    cond1 = True
    break

# test.add_assertion(2, "Load procnetdev2 (old style).")
cond2 = False
while True: # will break
    rc2, out2 = agg1.exec_run("ldmsd_controller -x sock -p 411 -h node-2 <<<plugn_status")

    if rc2 != 0:
        test.assert_test(2, False, f"plugn_status error, rc: {rc2}, out: {out2}")
        break
    l21,     = out2.splitlines()[2:]
    if not l21.startswith('procnetdev2 '):
        test.assert_test(2, False, f"Bad output: {out2}")
        break
    test.assert_test(2, True, "verified")
    cond2 = True
    break

cond = cond1 and cond2

# test.add_assertion(3, "Sets being created accordingly.")
while cond: # will break
    cond = False
    time.sleep(4)
    rc1, out1 = agg1.exec_run(f"/tada-src/python/ldms_ls.py -h agg-1 -l")
    data = json.loads(out1)
    data0 = data
    cond = verify_data(3, data,
                {
                    "node-1/netdev0": {
                        "data": {
                            "netdev_list": [
                                { "name" : "eth0" },
                                { "name" : "eth1" },
                            ],
                        }

                    },
                    "node-1/netdev1": {
                        "data": {
                            "netdev_list": [
                                { "name" : "lo" },
                            ],
                        }
                    },
                    "node-2/netdev2": {
                        "data": {
                            "netdev_list": [
                                { "name" : "lo" },
                                { "name" : "eth0" },
                                { "name" : "eth1" },
                            ],
                        }
                    }
                }
            )
    break

rc, out = node1.exec_run("ip link add dummy0 type dummy")
assert(rc == 0)
time.sleep(2)

# test.add_assertion(4, "Sets being updated accordingly (new network interface).")
while cond: # will break
    cond = False
    time.sleep(4)
    rc1, out1 = agg1.exec_run(f"/tada-src/python/ldms_ls.py -h agg-1 -l")
    data = json.loads(out1)
    data0 = data
    cond = verify_data(4, data,
                {
                    "node-1/netdev0": {
                        "data": {
                            "netdev_list": [
                                { "name" : "eth0" },
                                { "name" : "eth1" },
                            ],
                        }

                    },
                    "node-1/netdev1": {
                        "data": {
                            "netdev_list": [
                                { "name" : "lo" },
                                { "name" : "dummy0" },
                            ],
                        }
                    },
                    "node-2/netdev2": {
                        "data": {
                            "netdev_list": [
                                { "name" : "lo" },
                                { "name" : "eth0" },
                                { "name" : "eth1" },
                            ],
                        }
                    }
                }
            )
    break

rc, out = node1.exec_run("ip link del dummy0")
assert(rc == 0)
time.sleep(2)

# test.add_assertion(5, "Sets being updated accordingly (network interface removal).")
while cond: # will break
    cond = False
    time.sleep(4)
    rc1, out1 = agg1.exec_run(f"/tada-src/python/ldms_ls.py -h agg-1 -l")
    data = json.loads(out1)
    data0 = data
    cond = verify_data(5, data,
                {
                    "node-1/netdev0": {
                        "data": {
                            "netdev_list": [
                                { "name" : "eth0" },
                                { "name" : "eth1" },
                            ],
                        }

                    },
                    "node-1/netdev1": {
                        "data": {
                            "netdev_list": [
                                { "name" : "lo" },
                            ],
                        }
                    },
                    "node-2/netdev2": {
                        "data": {
                            "netdev_list": [
                                { "name" : "lo" },
                                { "name" : "eth0" },
                                { "name" : "eth1" },
                            ],
                        }
                    }
                }
            )
    break

# test.add_assertion(6, "The set is removed from node-1 when procnetdev2 terminated.")
# test.add_assertion(7, "The set is removed from agg-1 when procnetdev2 terminated.")
while cond:
    rc, out = node1.config_ldmsd("stop name=p0")
    if rc:
        test.assert_test(6, False, f"ldmsd `stop` error, rc: {rc}, out: {out}")
        break
    time.sleep(3)
    rc, out = node1.config_ldmsd("term name=p0")
    if rc:
        test.assert_test(6, False, f"ldmsd `term` error, rc: {rc}, out: {out}")
        break
    time.sleep(3)
    rc, out = agg1.exec_run(f"/tada-src/python/ldms_ls.py -h agg-1 -l")
    if rc:
        test.assert_test(6, False, f"ldms_ls.py error, rc: {rc}, out: {out}")
        break
    agg1_data = json.loads(out)
    rc, out = agg1.exec_run(f"/tada-src/python/ldms_ls.py -h node-1 -l")
    if rc:
        test.assert_test(6, False, f"ldms_ls.py error, rc: {rc}, out: {out}")
        break
    node1_data = json.loads(out)

    cond = verify_data(6, node1_data,
                {
                    "node-1/netdev1": {
                        "data": {
                            "netdev_list": [
                                { "name" : "lo" },
                            ],
                        }
                    },
                }
            )
    cond = verify_data(7, agg1_data,
                {
                    "node-1/netdev1": {
                        "data": {
                            "netdev_list": [
                                { "name" : "lo" },
                            ],
                        }
                    },
                    "node-2/netdev2": {
                        "data": {
                            "netdev_list": [
                                { "name" : "lo" },
                                { "name" : "eth0" },
                                { "name" : "eth1" },
                            ],
                        }
                    }
                }
            )
    break

# see `at_exit()` function