op-test

#!/usr/bin/python2
# IBM_PROLOG_BEGIN_TAG
# This is an automatically generated prolog.
#
# $Source: op-test-framework/ci/source/op_ci_bmc.py $
#
# OpenPOWER Automated Test Project
#
# Contributors Listed Below - COPYRIGHT 2015-2017
# [+] International Business Machines Corp.
#
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied. See the License for the specific language governing
# permissions and limitations under the License.
#
# IBM_PROLOG_END_TAG
"""
op-test: run OpenPOWER test suite(s)
"""
import sys
import os
import unittest

try:
  import faulthandler
  import signal
  faulthandler.register(signal.SIGUSR1)
except ImportError:
  pass

import OpTestConfiguration
from testcases import HelloWorld
from testcases import OpTestSwitchEndianSyscall
from testcases import OpTestSensors
from testcases import OpTestPrdDriver
from testcases import OpTestPCI
from testcases import FWTS
from testcases import BasicIPL
from testcases import PetitbootDropbearServer
from testcases import OpTestRTCdriver
from testcases import OpTestEM
from testcases import AT24driver
from testcases import OpTestEEH
from testcases import OpTestEnergyScale
from testcases import OpTestFastReboot
from testcases import OpTestHMIHandling
from testcases import OpTestHeartbeat
from testcases import I2C
from testcases import OpTestIPMILockMode
from testcases import OpTestIPMIReprovision
from testcases import OpTestInbandIPMI
from testcases import OpTestInbandUsbInterface
from testcases import OpTestNVRAM
from testcases import OpTestOOBIPMI
from testcases import OpTestOCC
from testcases import OpTestSystemBootSequence
from testcases import OpTestDumps
from testcases import HostLogin
from testcases import OpalMsglog
from testcases import KernelLog
from testcases import OpTestFlash
from testcases import OpalErrorLog
from testcases import LightPathDiagnostics
from testcases import DPO
from testcases import EPOW
from testcases import OpTestKernel
from testcases import fspresetReload
from testcases import OpTestPrdDaemon
from testcases import OpalUtils
from testcases import fspTODCorruption
from testcases import Console
from testcases import testRestAPI
from testcases import OpTestPNOR
from testcases import OpalGard
from testcases import SbePassThrough
from testcases import DeviceTreeValidation
from testcases import DeviceTreeWarnings
import testcases

args, remaining_args = OpTestConfiguration.conf.parse_args(sys.argv)
OpTestConfiguration.conf.objs()

print args

class StandbySuite():
    '''Machine at standby. Focused on BMC'''
    def __init__(self):
        self.s = unittest.TestSuite()
#        self.s.addTest(OpTestEnergyScale.standby_suite())
    def suite(self):
        return self.s

class SkirootSuite():
    '''Tests in Petitboot environment'''
    def __init__(self):
        self.s = unittest.TestSuite()
        self.s.addTest(DeviceTreeWarnings.Skiroot())
        self.s.addTest(OpTestEM.skiroot_suite())
        self.s.addTest(OpTestInbandIPMI.skiroot_full_suite())
        self.s.addTest(OpTestInbandUsbInterface.skiroot_full_suite())
        self.s.addTest(OpTestRTCdriver.SkirootRTC())
        self.s.addTest(AT24driver.SkirootAT24())
        self.s.addTest(I2C.BasicSkirootI2C())
        self.s.addTest(OpTestPCI.TestPCISkiroot())
        self.s.addTest(OpTestPCI.PcieLinkErrorsSkiroot())
        self.s.addTest(PetitbootDropbearServer.PetitbootDropbearServer())
        self.s.addTest(OpTestFastReboot.OpTestFastReboot())
        self.s.addTest(OpTestHeartbeat.HeartbeatSkiroot())
        self.s.addTest(OpTestNVRAM.SkirootNVRAM())
        self.s.addTest(Console.suite())
        self.s.addTest(OpTestPNOR.OpTestPNOR())
        self.s.addTest(DeviceTreeValidation.DeviceTreeValidationSkiroot())
        self.s.addTest(OpalMsglog.Skiroot())
        self.s.addTest(KernelLog.Skiroot())
        self.s.addTest(DPO.DPOSkiroot())
#        self.s.addTest(OpTestEnergyScale.runtime_suite())
    def suite(self):
        return self.s

class HostSuite():
    '''Tests run in booted OS'''
    def __init__(self):
        self.s = unittest.TestSuite()
        self.s.addTest(HostLogin.OOBHostLogin())
        self.s.addTest(DeviceTreeWarnings.Host())
        self.s.addTest(OpTestPrdDriver.OpTestPrdDriver())
        self.s.addTest(OpTestPCI.TestPCIHost())
        self.s.addTest(OpTestPCI.PcieLinkErrorsHost())
        self.s.addTest(FWTS.FWTS())
        self.s.addTest(OpTestRTCdriver.BasicRTC())
        self.s.addTest(OpTestEM.host_suite())
        self.s.addTest(AT24driver.AT24driver())
        self.s.addTest(I2C.BasicI2C())
        self.s.addTest(OpTestIPMILockMode.OpTestIPMILockMode())
        self.s.addTest(OpTestInbandIPMI.basic_suite())
        self.s.addTest(OpTestInbandUsbInterface.basic_suite())
        self.s.addTest(OpTestNVRAM.HostNVRAM())
        self.s.addTest(OpTestHeartbeat.HeartbeatHost())
        self.s.addTest(OpTestSensors.OpTestSensors())
        self.s.addTest(OpalErrorLog.BasicTest())
        self.s.addTest(OpTestPrdDaemon.OpTestPrdDaemon())
        self.s.addTest(SbePassThrough.SbePassThrough())
        self.s.addTest(DeviceTreeValidation.DeviceTreeValidationHost())
        self.s.addTest(OpalMsglog.Host())
        self.s.addTest(KernelLog.Host())
    def suite(self):
        return self.s

class ExperimentalSuite():
    '''Tests that need further development'''
    def __init__(self):
        self.s = unittest.TestSuite()
        self.s.addTest(OpTestEEH.suite())
        # SwitchEndian is here as we need to resolve issue of running
        # kernel self-tests as part of op-test or not.
        self.s.addTest(OpTestSwitchEndianSyscall.OpTestSwitchEndianSyscall())
    def suite(self):
        return self.s

class BasicIPLSuite():
    '''Basic boot/reboot power on/off'''
    def suite(self):
        return BasicIPL.suite()

class DefaultSuite():
    '''Basic regression tests'''
    def __init__(self):
        self.s = unittest.TestSuite()
        self.s.addTest(SkirootSuite().suite())
        self.s.addTest(HostSuite().suite())
    def suite(self):
        return self.s

class FullSuite():
    '''Every stable test'''
    def __init__(self):
        self.s = unittest.TestSuite()
        self.s.addTest(BasicIPLSuite().suite())
        self.s.addTest(DefaultSuite().suite())
        self.s.addTest(testRestAPI.RestAPI())
        self.s.addTest(OpalGard.OpalGard())
        self.s.addTest(OpalUtils.OpalUtils())
        self.s.addTest(OpTestRTCdriver.HostRTC())
        self.s.addTest(OpTestInbandIPMI.full_suite())
        self.s.addTest(OpTestInbandUsbInterface.full_suite())
        self.s.addTest(I2C.FullI2C())
        self.s.addTest(OpTestSensors.OpTestSensors())
        self.s.addTest(LightPathDiagnostics.suite())
        self.s.addTest(OpalErrorLog.FullTest())
        self.s.addTest(DPO.DPOHost())
        self.s.addTest(OpTestSwitchEndianSyscall.OpTestSwitchEndianSyscall())
        self.s.addTest(OpalMsglog.Host())
        self.s.addTest(KernelLog.Host())
    def suite(self):
        return self.s

class OpTestEMSuite():
    '''Energy Management'''
    def __init__(self):
        self.s = unittest.TestSuite()
    def suite(self):
        self.s.addTest(OpTestEM.host_suite())
        self.s.addTest(OpTestEM.skiroot_suite())
        return self.s

class BasicPCISuite():
    '''Basic PCI tests'''
    def suite(self):
        return OpTestPCI.suite()

class OpTestEEHSuite():
    '''PCI EEH error recovery'''
    def suite(self):
        return OpTestEEH.suite()

class HMISuite():
    '''HMI handling'''
    def suite(self):
        return OpTestHMIHandling.suite()

class ExperimentalHMISuite():
    def suite(self):
        return OpTestHMIHandling.experimental_suite()

class UnrecoverableHMISuite():
    def suite(self):
        return OpTestHMIHandling.unrecoverable_suite()

class OpTestEnergyScaleSuite():
    def __init__(self):
        self.s = unittest.TestSuite()
        self.s.addTest(OpTestEnergyScale.standby_suite())
        self.s.addTest(OpTestEnergyScale.runtime_suite())
    def suite(self):
        return self.s

class BrokenReprovisionSuite():
    def suite(self):
        return OpTestIPMIReprovision.broken_suite()

class ExperimentalReprovisionSuite():
    def suite(self):
        return OpTestIPMIReprovision.experimental_suite()

class InbandIPMISuite():
    def __init__(self):
        self.s = unittest.TestSuite()
        self.s.addTest(OpTestInbandIPMI.basic_suite())
        self.s.addTest(OpTestInbandIPMI.full_suite())
        self.s.addTest(OpTestInbandUsbInterface.basic_suite())
        self.s.addTest(OpTestInbandUsbInterface.full_suite())
        self.s.addTest(OpTestInbandIPMI.skiroot_basic_suite())
        self.s.addTest(OpTestInbandIPMI.skiroot_full_suite())
        self.s.addTest(OpTestInbandUsbInterface.skiroot_full_suite())
    def suite(self):
        return self.s


class OutofbandIPMISuite():
    '''Out of band IPMI'''
    def __init__(self):
        self.s = unittest.TestSuite()
        self.s.addTest(OpTestOOBIPMI.basic_suite())
        self.s.addTest(OpTestOOBIPMI.standby_suite())
        self.s.addTest(OpTestOOBIPMI.runtime_suite())
    def suite(self):
        return self.s

class OCCSuite():
    '''OCC Test Suite'''
    def __init__(self):
        self.s = unittest.TestSuite()
        self.s.addTest(OpTestOCC.OCC_RESET())
        self.s.addTest(OpTestOCC.basic_suite())
        self.s.addTest(OpTestOCC.full_suite())
        self.s.addTest(OpTestOCC.OCCRESET_FSP())
    def suite(self):
        return self.s

class SystemIPLSuite():
    '''System IPL's'''
    def suite(self):
        return OpTestSystemBootSequence.suite()

class CrashSuite():
    '''Crash Test Suite'''
    def suite(self):
        return OpTestKernel.crash_suite()

class FspOpalSuite():
    def __init__(self):
        self.s = unittest.TestSuite()
        self.s.addTest(OpTestDumps.suite())
        self.s.addTest(OpalErrorLog.BasicTest())
        self.s.addTest(OpalErrorLog.FullTest())
        self.s.addTest(LightPathDiagnostics.extended_suite())
        self.s.addTest(fspresetReload.suite())
        self.s.addTest(EPOW.suite())
        self.s.addTest(OpTestOCC.OCCRESET_FSP())
        self.s.addTest(fspTODCorruption.suite())
    def suite(self):
        return self.s

class KnownBugs():
    def __init__(self):
        self.s = unittest.TestSuite()
        self.s.addTest(Console.Console32k())
        self.s.addTest(Console.ControlC())
    def suite(self):
        return self.s

class FlashFirmware():
    def __init__(self):
        self.s = unittest.TestSuite()
        self.s.addTest(testcases.OpTestFlash.FSPFWImageFLASH())
        self.s.addTest(testcases.OpTestFlash.BmcImageFlash())
        self.s.addTest(testcases.OpTestFlash.PNORFLASH())
        self.s.addTest(testcases.OpTestFlash.OpalLidsFLASH())
    def suite(self):
        return self.s


suites = {
    'skiroot' : SkirootSuite(),
    'host'    : HostSuite(),
    'default' : DefaultSuite(),
    'BasicIPL' : BasicIPLSuite(),
    'BasicPCI' : BasicPCISuite(),
    'em' :       OpTestEMSuite(),
    'known-bugs' : KnownBugs(),
    'experimental': ExperimentalSuite(),
    'experimental-eeh': OpTestEEHSuite(),
    'experimental-energyscale' : OpTestEnergyScaleSuite(),
    'standby' : StandbySuite(),
    'hmi' : HMISuite(),
    'experimental-hmi' : ExperimentalHMISuite(),
    'experimental-unrecoverable-hmi' : UnrecoverableHMISuite(),
    'experimental-reprovision' : ExperimentalReprovisionSuite(),
    'broken-reprovision' : BrokenReprovisionSuite(),
    'full-inbandipmi' : InbandIPMISuite(),
    'full-outofbandipmi' : OutofbandIPMISuite(),
    'full-occ' : OCCSuite(),
    'crash-suite' : CrashSuite(),
    'system-ipl' : SystemIPLSuite(),
    'fsp-opal-suite' :  FspOpalSuite(),
    'full' : FullSuite(),
}

# Loop through the addons and load in suites defined there
for opt in OpTestConfiguration.optAddons:
    suites = OpTestConfiguration.optAddons[opt].addSuites(suites)

if OpTestConfiguration.conf.args.list_suites:
    print '{0:34}{1}'.format('Test Suite', 'Description')
    print '{0:34}{1}'.format('----------', '-----------')
    for key in suites:
        print '{0:34}{1}'.format(key, suites[key].__doc__)
    exit(0)

reload(sys)
sys.setdefaultencoding("utf8")

t = unittest.TestSuite()

if OpTestConfiguration.conf.args.run_suite:
    for suite in OpTestConfiguration.conf.args.run_suite:
        t.addTest(suites[suite].suite())

if OpTestConfiguration.conf.args.run:
    t.addTest(unittest.TestLoader().loadTestsFromNames(OpTestConfiguration.conf.args.run))

if not OpTestConfiguration.conf.args.run_suite and not OpTestConfiguration.conf.args.run:
    if not OpTestConfiguration.conf.args.only_flash:
        print "RUNNING DEFAULT SUITE"
        t.addTest(suites['default'].suite())

xml_msg = ""
def run_tests(t):
    try:
        print("Output to be written to: %s/*%s*" % (OpTestConfiguration.conf.output, OpTestConfiguration.conf.outsuffix))
        import xmlrunner  # requires unittest-xml-reporting package
        res = xmlrunner.XMLTestRunner(output=OpTestConfiguration.conf.output, outsuffix=OpTestConfiguration.conf.outsuffix, verbosity=2).run(t)
        print("Output written to: %s/*%s*" % (OpTestConfiguration.conf.output, OpTestConfiguration.conf.outsuffix))
        return res
    except ImportError, err:
        sys.stderr.write("WARNING: xmlrunner module not available, falling back to using unittest...\n\n")
        res = unittest.TextTestRunner(verbosity=2).run(t)
        return res

res = None
if not OpTestConfiguration.conf.args.noflash:
    res = run_tests(FlashFirmware().suite())

print repr(res)

if OpTestConfiguration.conf.args.only_flash:
    if res != None:
        exit(len(res.errors + res.failures))
    else:
        exit(0)

if not res or (res and not (res.errors or res.failures)):
    res = run_tests(t)
else:
    print "Skipping main tests as flashing failed"
    exit(-1)

exit(len(res.errors + res.failures))