Fixed PS kernel only xclbin download issue (#7674)

* Fixed PS kernel only xclbin download issue Updated ps_validate test to run the test Reset ps_slot_id to 0 after hot reset Signed-off-by: Jeff Lin <[email protected]> * Updated zocl to always cache xclbin regardless number of PS kernel sections. Also updated ps_bandwidth host code. Signed-off-by: Jeff Lin <[email protected]> * Minor update for PS kernel Signed-off-by: Jeff Lin <[email protected]> * Updated zocl_sysfs to only display slot 0's xclbin Added caching of AIE_METADATA Signed-off-by: Jeff Lin <[email protected]> * Fixed compile error for zocl_sysfs.c Signed-off-by: Jeff Lin <[email protected]> * Updated ps_bandwidth test dependency xclbin Signed-off-by: Jeff Lin <[email protected]> * Revert ps_bandwidth Signed-off-by: Jeff Lin <[email protected]> * Apply suggestions from code review Co-authored-by: Saifuddin Kaijar <[email protected]> * Move ps-pl-verify host code into XRT source Signed-off-by: Daniel Benusovich <[email protected]> * Removed unused variable from catch Signed-off-by: Daniel Benusovich <[email protected]> * Remove duplicate function declarations Signed-off-by: Daniel Benusovich <[email protected]> * Fixed aie xclbin switching issue Signed-off-by: Jeff Lin <[email protected]> --------- Signed-off-by: Jeff Lin <[email protected]> Signed-off-by: Daniel Benusovich <[email protected]> Co-authored-by: Jeff Lin <[email protected]> Co-authored-by: Saifuddin Kaijar <[email protected]> Co-authored-by: Daniel Benusovich <[email protected]> Co-authored-by: Daniel Benusovich <[email protected]>
Xilinx · Sep 8, 2023 · aa3449c · aa3449c
1 parent 2c3fbb6
commit aa3449c
Show file tree

Hide file tree

Showing 13 changed files with 190 additions and 52 deletions.
diff --git a/src/runtime_src/core/edge/drm/zocl/common/zocl_sysfs.c b/src/runtime_src/core/edge/drm/zocl/common/zocl_sysfs.c
@@ -3,6 +3,7 @@
  * A GEM style device manager for PCIe based OpenCL accelerators.
  *
  * Copyright (C) 2016-2022 Xilinx, Inc. All rights reserved.
+ * Copyright (C) 2023 Advanced Micro Devices, Inc. All rights reserved.
  *
  * Authors:
  *
@@ -598,27 +599,29 @@ static ssize_t read_xclbin_full(struct file *filp, struct kobject *kobj,
 
 	read_lock(&zdev->attr_rwlock);
 
-	for (i = 0; i < MAX_PR_SLOT_NUM; i++) {
-		zocl_slot = zdev->pr_slot[i];
-		if (!zocl_slot || !zocl_slot->axlf)
-			continue;
+	// Only read slot 0's xclbin - TODO: extend to multi-slot 
+	zocl_slot = zdev->pr_slot[0];
+	if (!zocl_slot || !zocl_slot->axlf) {
+		read_unlock(&zdev->attr_rwlock);
+		return 0;
+	}
 
-		size = zocl_slot->axlf_size;
-		if (off >= size) {
-			read_unlock(&zdev->attr_rwlock);
-			return 0;
-		}
+	size = zocl_slot->axlf_size;
+	if (off >= size) {
+		read_unlock(&zdev->attr_rwlock);
+		return 0;
+	}
 
-		if (count < size - off)
-			nread = count;
-		else
-			nread = size - off;
+	if (count < size - off)
+		nread = count;
+	else
+		nread = size - off;
 
-		memcpy(buf, ((char *)zocl_slot->axlf + off), nread);
+	memcpy(buf, ((char *)zocl_slot->axlf + off), nread);
+
+	buf += nread;
+	f_nread += nread;
 
-		buf += nread;
-		f_nread += nread;
-	}
 	read_unlock(&zdev->attr_rwlock);
 
 	return f_nread;

diff --git a/src/runtime_src/core/edge/drm/zocl/zert/zocl_ps_xclbin.c b/src/runtime_src/core/edge/drm/zocl/zert/zocl_ps_xclbin.c
@@ -164,7 +164,7 @@ zocl_load_pskernel(struct drm_zocl_dev *zdev, struct axlf *axlf, u32 slot_idx)
  */
 static int
 zocl_kernel_cache_xclbin(struct drm_zocl_dev *zdev, struct drm_zocl_slot *slot,
-			 struct axlf *axlf, char *xclbin_ptr)
+			 struct axlf *axlf)
 {
 	size_t size = axlf->m_header.m_length;
 	struct axlf *slot_axlf = NULL;
@@ -175,7 +175,7 @@ zocl_kernel_cache_xclbin(struct drm_zocl_dev *zdev, struct drm_zocl_slot *slot,
 		return -ENOMEM;
 	}
 
-	memcpy(slot_axlf, xclbin_ptr, size);
+	memcpy(slot_axlf, axlf, size);
 
 	write_lock(&zdev->attr_rwlock);
 	slot->axlf = slot_axlf;
@@ -257,7 +257,7 @@ zocl_xclbin_load_pdi(struct drm_zocl_dev *zdev, void *data,
 	count = xrt_xclbin_get_section_num(axlf, SOFT_KERNEL);
 	if (count > 0) {
 		/* SAIF TODO */
-		ret = zocl_kernel_cache_xclbin(zdev, slot, axlf, xclbin);
+		ret = zocl_kernel_cache_xclbin(zdev, slot, axlf);
 		if (ret) {
 			DRM_ERROR("%s cannot cache xclbin",__func__);
 			goto out;
@@ -304,6 +304,8 @@ zocl_xclbin_load_pskernel(struct drm_zocl_dev *zdev, void *data, uint32_t slot_i
 	void *aie_res = 0;
 	struct device_node *aienode = NULL;
 	uint8_t hw_gen = 1;
+	struct aie_metadata      aie_data = { 0 };
+	uint64_t size = 0;
 
         if (memcmp(axlf_head->m_magic, "xclbin2", 8)) {
                 DRM_INFO("Invalid xclbin magic string");
@@ -328,8 +330,6 @@ zocl_xclbin_load_pskernel(struct drm_zocl_dev *zdev, void *data, uint32_t slot_i
 	size_of_header = sizeof(struct axlf_section_header);
 	num_of_sections = axlf_head->m_header.m_numSections-1;
 	xclbin = (char *)axlf;
-	if (zocl_xclbin_get_uuid(slot) != NULL)
-		zocl_cleanup_aie(zdev);
 
 	/*
 	 * Read AIE_RESOURCES section. aie_res will be NULL if there is no
@@ -349,18 +349,36 @@ zocl_xclbin_load_pskernel(struct drm_zocl_dev *zdev, void *data, uint32_t slot_i
 		aienode = NULL;
 	}
 
+	/*
+	 * Read AIE_METADATA section
+	 */
+	count = xrt_xclbin_get_section_num(axlf, AIE_METADATA);
+	if(count > 0) {
+		size = zocl_read_sect_kernel(AIE_METADATA, &aie_data.data, axlf, xclbin);
+		aie_data.size = size;
+	}
+	slot->aie_data = aie_data;
+
+	/* Mark AIE out of reset state after load PDI */
+	if (zdev->aie) {
+		mutex_lock(&zdev->aie_lock);
+		zdev->aie->aie_reset = false;
+		mutex_unlock(&zdev->aie_lock);
+	}
+
 	// Cache full xclbin
 	//last argument represents aie generation. 1. aie, 2. aie-ml ...
 	DRM_INFO("AIE Device set to gen %d", hw_gen);
 	zocl_create_aie(zdev, axlf, aie_res, hw_gen);
 
-	count = xrt_xclbin_get_section_num(axlf, SOFT_KERNEL);
+
+	ret = zocl_kernel_cache_xclbin(zdev, slot, axlf);
+	if (ret) {
+		DRM_ERROR("%s cannot cache xclbin",__func__);
+		goto out;
+	}
+        count = xrt_xclbin_get_section_num(axlf, SOFT_KERNEL);
 	if (count > 0) {
-		ret = zocl_kernel_cache_xclbin(zdev, slot, axlf, xclbin);
-		if (ret) {
-			DRM_ERROR("%s cannot cache xclbin",__func__);
-			goto out;
-		}
 		ret = zocl_load_pskernel(zdev, slot->axlf, slot_id);
 		if (ret)
 			goto out;

diff --git a/src/runtime_src/core/edge/drm/zocl/zert/zocl_sk.c b/src/runtime_src/core/edge/drm/zocl/zert/zocl_sk.c
@@ -27,6 +27,7 @@ zocl_sk_getcmd_ioctl(struct drm_device *dev, void *data, struct drm_file *filp)
 	struct soft_krnl *sk = zdev->soft_kernel;
 	struct soft_krnl_cmd *scmd;
 	struct drm_zocl_sk_getcmd *kdata = data;
+	xuid_t *phy_slot_uuid = NULL;
 
 	if (!sk)
 		return 0;
@@ -62,16 +63,20 @@ zocl_sk_getcmd_ioctl(struct drm_device *dev, void *data, struct drm_file *filp)
 		for (i = 0;i < MAX_PR_SLOT_NUM;i++) {
 			mutex_lock(&zdev->pr_slot[i]->slot_xclbin_lock);
 			slot_uuid = zocl_xclbin_get_uuid(zdev->pr_slot[i]);
-			mutex_unlock(&zdev->pr_slot[i]->slot_xclbin_lock);
-			if(uuid_equal(slot_uuid,(xuid_t *)cmd->sk_uuid)) {
-				slot_id = i;
-				break;
+			if(slot_uuid) {
+				mutex_unlock(&zdev->pr_slot[i]->slot_xclbin_lock);
+				if(uuid_equal(slot_uuid,(xuid_t *)cmd->sk_uuid)) {
+					slot_id = i;
+					break;
+				}
 			}
 		}
 
 		if (slot_id == MAX_PR_SLOT_NUM) {
-			DRM_ERROR("PS Kernel UUID %s not found!",cmd->sk_uuid);
+			DRM_ERROR("PS Kernel UUID %lx not found!",cmd->sk_uuid);
 			return -EINVAL;
+		} else {
+			DRM_INFO("PS Kernel UUID %lx found at slot %d\n",cmd->sk_uuid, slot_id);
 		}
 
 		if (sk->sk_meta_bohdl[slot_id] >= 0) {
@@ -124,7 +129,11 @@ zocl_sk_getcmd_ioctl(struct drm_device *dev, void *data, struct drm_file *filp)
 		kdata->cu_nums = cmd->num_cus;
 		kdata->bohdl = bohdl;
 		kdata->meta_bohdl = meta_bohdl;
-		memcpy(kdata->uuid,cmd->sk_uuid,sizeof(kdata->uuid));
+		// Pass physical slot 0 UUID to SKD - Currently we only support 1 physical slot
+		mutex_lock(&zdev->pr_slot[0]->slot_xclbin_lock);
+		phy_slot_uuid = zocl_xclbin_get_uuid(zdev->pr_slot[0]);
+		mutex_unlock(&zdev->pr_slot[0]->slot_xclbin_lock);
+		memcpy(kdata->uuid,phy_slot_uuid,sizeof(kdata->uuid));
 
 		snprintf(kdata->name, ZOCL_MAX_NAME_LENGTH, "%s",
 		    (char *)cmd->sk_name);

diff --git a/src/runtime_src/core/edge/ps_kernels/xrt/tests/validate/test_dependencies.json b/src/runtime_src/core/edge/ps_kernels/xrt/tests/validate/test_dependencies.json
@@ -6,7 +6,7 @@
     },
     {
       "name": "ps_bandwidth.xclbin",
-      "dependencies": ["ps_bandwidth.xclbin"]
+      "dependencies": ["bandwidth.xclbin"]
     },
     {
       "name": "ps_validate.xclbin",

diff --git a/src/runtime_src/core/edge/skd/xrt_skd.cpp b/src/runtime_src/core/edge/skd/xrt_skd.cpp
@@ -132,7 +132,7 @@ namespace xrt {
     if (kernel_init) {
 	ret = xrtDeviceLoadXclbinUUID(m_xrtdhdl, m_xclbin_uuid.get());
 	if (ret) {
-	    const auto errMsg = boost::format("Cannot load xclbin from UUID!  UUID = %s") % m_xclbin_uuid.get();
+	    const auto errMsg = boost::format("Cannot load xclbin from UUID!  UUID = %s") % m_xclbin_uuid.to_string();
 	    xrt_core::message::send(severity_level::error, "SKD", errMsg.str());
 	    return ret;
 	}

diff --git a/src/runtime_src/core/pcie/driver/linux/xocl/userpf/xocl_ioctl.c b/src/runtime_src/core/pcie/driver/linux/xocl/userpf/xocl_ioctl.c
@@ -808,6 +808,7 @@ int xocl_hot_reset_ioctl(struct drm_device *dev, void *data,
 	if (chan_disable & (1 << XCL_MAILBOX_REQ_HOT_RESET))
 		return -EOPNOTSUPP;
 
+	xdev->ps_slot_id = 0;  // Clear PS kernel xclbin slots after reset
 	xocl_drvinst_set_offline(xdev->core.drm, true);
 	xocl_queue_work(xdev, XOCL_WORK_RESET, XOCL_RESET_DELAY);
 	xocl_xdev_info(xdev, "Scheduled reset");

diff --git a/src/runtime_src/core/pcie/driver/linux/xocl/xocl_xclbin.c b/src/runtime_src/core/pcie/driver/linux/xocl/xocl_xclbin.c
@@ -247,6 +247,7 @@ int xocl_xclbin_download(xdev_handle_t xdev, const void *xclbin, uint32_t slot_i
 {
 	int rval = 0;
 
+	xocl_info(&XDEV(xdev)->pdev->dev,"slot_id = %d", slot_id);
 	if (XOCL_DSA_IS_VERSAL(xdev)) {
 		rval = xocl_xclbin_download_impl(xdev, xclbin, slot_id, &xgq_ops);
 		/* Legacy shell doesn't have xgq resources */

diff --git a/src/runtime_src/core/tools/common/TestRunner.cpp b/src/runtime_src/core/tools/common/TestRunner.cpp
@@ -77,9 +77,9 @@ TestRunner::TestRunner (const std::string & test_name,
                         const std::string & description,
                         const std::string & xclbin,
                         bool is_explicit)
-    : m_name(test_name)
+    : m_xclbin(xclbin)
+    , m_name(test_name)
     , m_description(description) 
-    , m_xclbin(xclbin)
     , m_explicit(is_explicit)
 {
   //Empty
@@ -367,6 +367,16 @@ TestRunner::runTestCase( const std::shared_ptr<xrt_core::device>& _dev, const st
   }
 }
 
+int
+TestRunner::validate_binary_file(const std::string& binaryfile)
+{
+  std::ifstream infile(binaryfile);
+  if (!infile.good()) 
+    return EOPNOTSUPP;
+  else
+    return EXIT_SUCCESS;
+}
+
 bool
 TestRunner::search_and_program_xclbin(const std::shared_ptr<xrt_core::device>& dev, boost::property_tree::ptree& ptTest)
 {

diff --git a/src/runtime_src/core/tools/common/TestRunner.h b/src/runtime_src/core/tools/common/TestRunner.h
@@ -47,26 +47,27 @@ class TestRunner {
              boost::property_tree::ptree& _ptTest);
     void logger(boost::property_tree::ptree& ptree, const std::string& tag, const std::string& msg);
     bool search_and_program_xclbin(const std::shared_ptr<xrt_core::device>& dev, boost::property_tree::ptree& ptTest);
+    std::string findPlatformPath(const std::shared_ptr<xrt_core::device>& _dev,
+                  boost::property_tree::ptree& _ptTest);
+    std::vector<std::string> findDependencies( const std::string& test_path,
+                      const std::string& ps_kernel_name);
+    int validate_binary_file(const std::string& binaryfile);
 
     const std::string test_token_skipped = "SKIPPED";
     const std::string test_token_failed = "FAILED";
     const std::string test_token_passed = "PASSED";
+    std::string m_xclbin;
 
   private:
     std::string searchLegacyXclbin(const uint16_t vendor, const std::string& dev_name, 
                       boost::property_tree::ptree& _ptTest);
     std::string searchSSV2Xclbin(const std::string& logic_uuid,
                       boost::property_tree::ptree& _ptTest);
-    std::string findPlatformPath(const std::shared_ptr<xrt_core::device>& _dev,
-                      boost::property_tree::ptree& _ptTest);
-    std::vector<std::string> findDependencies( const std::string& test_path,
-                      const std::string& ps_kernel_name);
 
   //variables
   private:
     std::string m_name;
     std::string m_description;
-    std::string m_xclbin;
     bool m_explicit;
 
 };