Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

TestRunner code cleanup #8571

Open
wants to merge 3 commits into
base: master
Choose a base branch
from
Open

TestRunner code cleanup #8571

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
ed0423a
TestRunner code cleanup
Oct 25, 2024
bcaa4a3
Removing some extra code
Oct 25, 2024
6cabce2
Removing some extra code
Oct 25, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
242 changes: 25 additions & 217 deletions src/runtime_src/core/tools/common/TestRunner.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@
// ------ I N C L U D E F I L E S -------------------------------------------
// Local - Include Files
#include "TestRunner.h"
#include "tests/TestValidateUtilities.h"
#include "core/common/error.h"
#include "core/common/module_loader.h"
#include "core/tools/common/Process.h"
Expand All @@ -20,7 +21,6 @@ namespace xq = xrt_core::query;
// System - Include Files
#include <fstream>
#include <iostream>
#include <regex>
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

thanks for cleaning up includes as well!

#include <thread>

static constexpr std::chrono::seconds max_test_duration = std::chrono::seconds(60 * 5); //5 minutes
Expand All @@ -39,25 +39,6 @@ runTestInternal(std::shared_ptr<xrt_core::device> dev,
is_thread_running = false;
}

static const std::string
getXsaPath(const uint16_t vendor)
{
if (vendor == 0 || (vendor == INVALID_ID))
return std::string();

std::string vendorName;
switch (vendor) {
case ARISTA_ID:
vendorName = "arista";
break;
default:
case XILINX_ID:
vendorName = "xilinx";
break;
}
return "/opt/" + vendorName + "/xsa/";
}

static void
program_xclbin(const std::shared_ptr<xrt_core::device>& device, const std::string& xclbin)
{
Expand Down Expand Up @@ -113,145 +94,7 @@ TestRunner::startTest(std::shared_ptr<xrt_core::device> dev)
return result;
}

/*
* mini logger to log errors, warnings and details produced by the test cases
*/
void
TestRunner::logger(boost::property_tree::ptree& _ptTest, const std::string& tag, const std::string& msg)
{
boost::property_tree::ptree _ptLog;
boost::property_tree::ptree _ptExistingLog;
boost::optional<boost::property_tree::ptree&> _ptChild = _ptTest.get_child_optional("log");
if (_ptChild)
_ptExistingLog = _ptChild.get();

_ptLog.put(tag, msg);
_ptExistingLog.push_back(std::make_pair("", _ptLog));
_ptTest.put_child("log", _ptExistingLog);
}

/*
* search for xclbin for an SSV2 platform
*/
std::string
TestRunner::searchSSV2Xclbin(const std::string& logic_uuid,
boost::property_tree::ptree& _ptTest)
{
#ifdef XRT_INSTALL_PREFIX
#define FW_DIR XRT_INSTALL_PREFIX "/firmware/"
#else
#define FW_DIR "/opt/xilinx/firmware/"
#endif
std::string formatted_fw_path(FW_DIR);
std::filesystem::path fw_dir(formatted_fw_path);
if (!std::filesystem::is_directory(fw_dir)) {
logger(_ptTest, "Error", boost::str(boost::format("Failed to find %s") % fw_dir));
logger(_ptTest, "Error", "Please check if the platform package is installed correctly");
_ptTest.put("status", test_token_failed);
return "";
}

std::vector<std::string> suffix = { "dsabin", "xsabin" };

for (const std::string& t : suffix) {
std::regex e("(^" + formatted_fw_path + "[^/]+/[^/]+/[^/]+/).+\\." + t);
for (std::filesystem::recursive_directory_iterator iter(fw_dir), end; iter != end;) {
std::string name = iter->path().string();
std::smatch cm;
if (!std::filesystem::is_directory(std::filesystem::path(name.c_str()))) {
iter.disable_recursion_pending();
}

std::regex_match(name, cm, e);
if (cm.size() > 0) {
auto dtbbuf = XBUtilities::get_axlf_section(name, PARTITION_METADATA);
if (dtbbuf.empty()) {
++iter;
continue;
}
std::vector<std::string> uuids = XBUtilities::get_uuids(dtbbuf.data());
if (!uuids.size()) {
++iter;
}
else if (uuids[0].compare(logic_uuid) == 0) {
return cm.str(1) + "test/";
}
}
else if (iter.depth() > 4) {
iter.pop();
continue;
}
++iter;
}
}
logger(_ptTest, "Details", boost::str(boost::format("Platform path not available. Skipping validation")));
_ptTest.put("status", test_token_skipped);
return "";
}

/*
* search for xclbin for a legacy platform
*/
std::string
TestRunner::searchLegacyXclbin(const uint16_t vendor, const std::string& dev_name,
boost::property_tree::ptree& _ptTest)
{
#ifdef XRT_INSTALL_PREFIX
#define DSA_DIR XRT_INSTALL_PREFIX "/dsa/"
#else
#define DSA_DIR "/opt/xilinx/dsa/"
#endif
const std::string dsapath(DSA_DIR);
const std::string xsapath(getXsaPath(vendor));

if (!std::filesystem::is_directory(dsapath) && !std::filesystem::is_directory(xsapath)) {
const auto fmt = boost::format("Failed to find '%s' or '%s'") % dsapath % xsapath;
logger(_ptTest, "Error", boost::str(fmt));
logger(_ptTest, "Error", "Please check if the platform package is installed correctly");
_ptTest.put("status", test_token_failed);
XBUtilities::throw_cancel(fmt);
}

//create possible xclbin paths
std::string xsaXclbinPath = xsapath + dev_name + "/test/";
std::string dsaXclbinPath = dsapath + dev_name + "/test/";
std::filesystem::path xsa_xclbin(xsaXclbinPath);
std::filesystem::path dsa_xclbin(dsaXclbinPath);
if (std::filesystem::exists(xsa_xclbin))
return xsaXclbinPath;
else if (std::filesystem::exists(dsa_xclbin))
return dsaXclbinPath;

const std::string fmt = "Platform path not available. Skipping validation";
logger(_ptTest, "Details", fmt);
_ptTest.put("status", test_token_skipped);
XBUtilities::throw_cancel(fmt);
return "";
}

/**
* @deprecated
* This function should be used ONLY for any legacy devices. IE versal/alveo only.
* Ideally this will be removed soon.
*
* Children of TestRunner should call into findPlatformFile
*/
std::string
TestRunner::findXclbinPath(const std::shared_ptr<xrt_core::device>& dev,
boost::property_tree::ptree& ptTest)
{
const std::string xclbin_name = ptTest.get<std::string>("xclbin", "");
const auto config_dir = ptTest.get<std::string>("xclbin_directory", "");
const std::filesystem::path platform_path = !config_dir.empty() ? config_dir : findPlatformPath(dev, ptTest);

const auto xclbin_path = platform_path / xclbin_name;
if (std::filesystem::exists(xclbin_path))
return xclbin_path.string();

logger(ptTest, "Details", boost::str(boost::format("%s not available. Skipping validation.") % xclbin_name));
ptTest.put("status", test_token_skipped);
return "";
}

/*
* helper funtion for kernel and bandwidth python test cases when there is no platform.json
Expand All @@ -266,21 +109,21 @@ TestRunner::runPyTestCase( const std::shared_ptr<xrt_core::device>& _dev, const
boost::property_tree::ptree& _ptTest)
{
const auto xclbin = _ptTest.get<std::string>("xclbin", "");
const auto xclbin_path = std::filesystem::path(findXclbinPath(_dev, _ptTest));
const auto xclbin_path = std::filesystem::path(XBValidateUtils::findXclbinPath(_dev, _ptTest));

// 0RP (nonDFX) flat shell support.
// Currently, there isn't a clean way to determine if a nonDFX shell's interface is truly flat.
// At this time, this is determined by whether or not it delivers an accelerator (e.g., verify.xclbin)
const auto logic_uuid = xrt_core::device_query_default<xq::logic_uuids>(_dev, {});
if (!logic_uuid.empty() && !std::filesystem::exists(xclbin_path)) {
logger(_ptTest, "Details", "Verify xclbin not available or shell partition is not programmed. Skipping validation.");
_ptTest.put("status", test_token_skipped);
XBValidateUtils::logger(_ptTest, "Details", "Verify xclbin not available or shell partition is not programmed. Skipping validation.");
_ptTest.put("status", XBValidateUtils::test_token_skipped);
return;
}

logger(_ptTest, "Xclbin", xclbin_path.parent_path().string());
XBValidateUtils::logger(_ptTest, "Xclbin", xclbin_path.parent_path().string());

std::string platform_path = findPlatformPath(_dev, _ptTest);
std::string platform_path = XBValidateUtils::findPlatformPath(_dev, _ptTest);

// Some testcases require additional binaries to be present on the device
std::string dependency_args;
Expand All @@ -301,13 +144,13 @@ TestRunner::runPyTestCase( const std::shared_ptr<xrt_core::device>& _dev, const
std::string xrtTestCasePath = XRT_TEST_CASE_DIR + py;
std::filesystem::path xrt_path(xrtTestCasePath);
if (!std::filesystem::exists(xrt_path)) {
logger(_ptTest, "Error", boost::str(boost::format("Failed to find %s") % xrtTestCasePath));
logger(_ptTest, "Error", "Please check if the platform package is installed correctly");
_ptTest.put("status", test_token_failed);
XBValidateUtils::logger(_ptTest, "Error", boost::str(boost::format("Failed to find %s") % xrtTestCasePath));
XBValidateUtils::logger(_ptTest, "Error", "Please check if the platform package is installed correctly");
_ptTest.put("status", XBValidateUtils::test_token_failed);
return;
}
// log testcase path for debugging purposes
logger(_ptTest, "Testcase", xrtTestCasePath);
XBValidateUtils::logger(_ptTest, "Testcase", xrtTestCasePath);

std::vector<std::string> args = { "-k", xclbin_path.string(),
"-d", xq::pcie_bdf::to_string(xrt_core::device_query<xq::pcie_bdf>(_dev)) };
Expand All @@ -316,19 +159,19 @@ TestRunner::runPyTestCase( const std::shared_ptr<xrt_core::device>& _dev, const
exit_code = XBU::runScript("python", xrtTestCasePath, args, os_stdout, os_stderr);

if (exit_code == EOPNOTSUPP) {
_ptTest.put("status", test_token_skipped);
_ptTest.put("status", XBValidateUtils::test_token_skipped);
}
else if (exit_code == EXIT_SUCCESS) {
_ptTest.put("status", test_token_passed);
_ptTest.put("status", XBValidateUtils::test_token_passed);
}
else {
logger(_ptTest, "Error", os_stdout.str());
logger(_ptTest, "Error", os_stderr.str());
_ptTest.put("status", test_token_failed);
XBValidateUtils::logger(_ptTest, "Error", os_stdout.str());
XBValidateUtils::logger(_ptTest, "Error", os_stderr.str());
_ptTest.put("status", XBValidateUtils::test_token_failed);
}
} catch (const std::exception& e) {
logger(_ptTest, "Error", e.what());
_ptTest.put("status", test_token_failed);
XBValidateUtils::logger(_ptTest, "Error", e.what());
_ptTest.put("status", XBValidateUtils::test_token_failed);
}

// Get out max thruput for bandwidth testcase
Expand All @@ -337,14 +180,14 @@ TestRunner::runPyTestCase( const std::shared_ptr<xrt_core::device>& _dev, const
size_t st = os_stdout.str().find("Maximum");
if (st != std::string::npos) {
size_t end = os_stdout.str().find("\n", st);
logger(_ptTest, "Details", os_stdout.str().substr(st, end - st));
XBValidateUtils::logger(_ptTest, "Details", os_stdout.str().substr(st, end - st));
}
else {
// new test cases to find "Throughput (Type: {...}) (Bank count: {...}):"
auto str = os_stdout.str().find("Throughput", 0);
while(str != std::string::npos) {
auto end = os_stdout.str().find("\n", str);
logger(_ptTest, "Details", os_stdout.str().substr(str, end - str));
XBValidateUtils::logger(_ptTest, "Details", os_stdout.str().substr(str, end - str));
str = os_stdout.str().find("Throughput" , end);
}
}
Expand All @@ -366,14 +209,14 @@ TestRunner::search_and_program_xclbin(const std::shared_ptr<xrt_core::device>& d
xuid_t uuid;
uuid_parse(xrt_core::device_query<xq::xclbin_uuid>(dev).c_str(), uuid);

const std::string xclbin_path = findXclbinPath(dev, ptTest);
const std::string xclbin_path = XBValidateUtils::findXclbinPath(dev, ptTest);

try {
program_xclbin(dev, xclbin_path);
}
catch (const std::exception& e) {
logger(ptTest, "Error", e.what());
ptTest.put("status", test_token_failed);
XBValidateUtils::logger(ptTest, "Error", e.what());
ptTest.put("status", XBValidateUtils::test_token_failed);
return false;
}

Expand All @@ -400,8 +243,8 @@ TestRunner::dpu_or_elf(const std::shared_ptr<xrt_core::device>& dev, const xrt::
if (itr!=xkernels.end())
xkernel = *itr;
else {
logger(ptTest, "Error", "No kernel with `DPU` found in the xclbin");
ptTest.put("status", test_token_failed);
XBValidateUtils::logger(ptTest, "Error", "No kernel with `DPU` found in the xclbin");
ptTest.put("status", XBValidateUtils::test_token_failed);
}
auto kernelName = xkernel.get_name();

Expand All @@ -410,7 +253,7 @@ TestRunner::dpu_or_elf(const std::shared_ptr<xrt_core::device>& dev, const xrt::
else {
// Elf flow
const auto elf_name = xrt_core::device_query<xrt_core::query::elf_name>(dev, xrt_core::query::elf_name::type::nop);
auto elf_path = findPlatformFile(elf_name, ptTest);
auto elf_path = XBValidateUtils::findPlatformFile(elf_name, ptTest);

return elf_path;
}
Expand All @@ -434,41 +277,6 @@ TestRunner::get_kernel(const xrt::hw_context& hwctx, const std::string& kernel_o
}
}

/**
* @deprecated
* This function should be used ONLY for any legacy devices. IE versal/alveo only.
* Ideally this will be removed soon.
*
* Children of TestRunner should call into findPlatformFile
*/
std::string
TestRunner::findPlatformPath(const std::shared_ptr<xrt_core::device>& dev,
boost::property_tree::ptree& ptTest)
{
const auto logic_uuid = xrt_core::device_query_default<xq::logic_uuids>(dev, {});
if (!logic_uuid.empty())
return searchSSV2Xclbin(logic_uuid.front(), ptTest);
else {
auto vendor = xrt_core::device_query<xq::pcie_vendor>(dev);
auto name = xrt_core::device_query<xq::rom_vbnv>(dev);
return searchLegacyXclbin(vendor, name, ptTest);
}
}

std::string
TestRunner::findPlatformFile(const std::string& file_path,
boost::property_tree::ptree& ptTest)
{
try {
return xrt_core::environment::platform_path(file_path).string();
}
catch (const std::exception&) {
logger(ptTest, "Details", boost::str(boost::format("%s not available") % file_path));
ptTest.put("status", test_token_skipped);
return "";
}
}

std::vector<std::string>
TestRunner::findDependencies( const std::string& test_path,
const std::string& ps_kernel_name)
Expand Down
14 changes: 0 additions & 14 deletions src/runtime_src/core/tools/common/TestRunner.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -40,30 +40,16 @@ class TestRunner : public JSONConfigurable {
const std::string & xclbin = "", bool is_explicit = false);
void runPyTestCase( const std::shared_ptr<xrt_core::device>& _dev, const std::string& py,
boost::property_tree::ptree& _ptTest);
void logger(boost::property_tree::ptree& ptree, const std::string& tag, const std::string& msg);
bool search_and_program_xclbin(const std::shared_ptr<xrt_core::device>& dev, boost::property_tree::ptree& ptTest);
std::string findPlatformPath(const std::shared_ptr<xrt_core::device>& dev, boost::property_tree::ptree& ptTest);
std::string findPlatformFile(const std::string& file_path, boost::property_tree::ptree& ptTest);
std::string findXclbinPath(const std::shared_ptr<xrt_core::device>& dev,
boost::property_tree::ptree& ptTest);
std::vector<std::string> findDependencies( const std::string& test_path,
const std::string& ps_kernel_name);
int validate_binary_file(const std::string& binaryfile);
std::string dpu_or_elf(const std::shared_ptr<xrt_core::device>& dev, const xrt::xclbin& xclbin,
boost::property_tree::ptree& ptTest);
xrt::kernel get_kernel(const xrt::hw_context& hwctx, const std::string& kernel_or_elf);

const std::string test_token_skipped = "SKIPPED";
const std::string test_token_failed = "FAILED";
const std::string test_token_passed = "PASSED";
std::string m_xclbin;

private:
std::string searchLegacyXclbin(const uint16_t vendor, const std::string& dev_name,
boost::property_tree::ptree& _ptTest);
std::string searchSSV2Xclbin(const std::string& logic_uuid,
boost::property_tree::ptree& _ptTest);

//variables
private:
xrt_core::query::xclbin_name::type m_xclbin_type;
Expand Down
Loading
Loading