Skip to content

Commit

Permalink
Restore information about Blob layout for new plugins (openvinotoolki…
Browse files Browse the repository at this point in the history 
…t#18696)

* Restore information about Blob layout for new plugins

* Save legacy names
  • Loading branch information
@ilyachur
ilyachur authored Jul 21, 2023
1 parent e076ed4 commit 44cae12
Show file tree
Hide file tree
Showing 5 changed files with 81 additions and 48 deletions.
4 changes: 3 additions & 1 deletion src/inference/dev_api/openvino/runtime/make_tensor.hpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -71,7 +71,9 @@ ov::SoPtr<ITensor> make_tensor(const std::shared_ptr<InferenceEngine::Blob>& ten
const InferenceEngine::Blob* get_hardware_blob(const InferenceEngine::Blob* blob);
InferenceEngine::Blob* get_hardware_blob(InferenceEngine::Blob* blob);

std::shared_ptr<InferenceEngine::Blob> tensor_to_blob(const ov::SoPtr<ITensor>& tensor, bool unwrap = true);
std::shared_ptr<InferenceEngine::Blob> tensor_to_blob(const ov::SoPtr<ITensor>& tensor,
bool unwrap = true,
InferenceEngine::TensorDesc desc = {});
/** @endcond */

IE_SUPPRESS_DEPRECATED_END
Expand Down
20 changes: 17 additions & 3 deletions src/inference/src/dev/converter_utils.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -544,14 +544,28 @@ class IInferRequestInternalWrapper : public InferenceEngine::IInferRequestIntern
}

InferenceEngine::Blob::Ptr GetBlob(const std::string& name) override {
return tensor_to_blob(m_request->get_tensor(find_port(name)));
auto port = find_port(name);
auto& rt_info = port.get_rt_info();
auto it = rt_info.find("ie_legacy_td");
InferenceEngine::TensorDesc desc;
if (it != rt_info.end()) {
desc = it->second.as<InferenceEngine::TensorDesc>();
}
return tensor_to_blob(m_request->get_tensor(port), true, desc);
}

InferenceEngine::BatchedBlob::Ptr GetBlobs(const std::string& name) override {
auto tensors = m_request->get_tensors(find_port(name));
auto port = find_port(name);
auto& rt_info = port.get_rt_info();
auto it = rt_info.find("ie_legacy_td");
InferenceEngine::TensorDesc desc;
if (it != rt_info.end()) {
desc = it->second.as<InferenceEngine::TensorDesc>();
}
auto tensors = m_request->get_tensors(port);
std::vector<InferenceEngine::Blob::Ptr> blobs;
for (const auto& tensor : tensors) {
blobs.emplace_back(tensor_to_blob(tensor));
blobs.emplace_back(tensor_to_blob(tensor, true, desc));
}
return std::make_shared<InferenceEngine::BatchedBlob>(blobs);
}
Expand Down
85 changes: 42 additions & 43 deletions src/inference/src/dev/make_tensor.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -441,11 +441,7 @@ class RemoteBlobTensor : public IRemoteTensor {
*/
class TensorRemoteBlob : public ie::RemoteBlob {
public:
TensorRemoteBlob(const ov::SoPtr<ITensor>& tensor)
: ie::RemoteBlob{ie::TensorDesc{ie::details::convertPrecision(tensor->get_element_type()),
tensor->get_shape(),
ie::TensorDesc::getLayoutByRank(tensor->get_shape().size())}},
tensor{tensor} {
TensorRemoteBlob(const ov::SoPtr<ITensor>& tensor, ie::TensorDesc desc) : ie::RemoteBlob{desc}, tensor{tensor} {
OPENVINO_ASSERT(this->tensor);
}
std::shared_ptr<ov::IRemoteTensor> cast_tensor() const {
Expand Down Expand Up @@ -508,38 +504,8 @@ template <typename T>
class TensorMemoryBlob : public ie::TBlob<T> {
public:
~TensorMemoryBlob() override = default;
explicit TensorMemoryBlob(const ov::SoPtr<ITensor>& tensor_) try : ie
::TBlob<T>{[&] {
auto element_type = tensor_->get_element_type();
auto shape = tensor_->get_shape();
ie::SizeVector blk_order(shape.size());
std::iota(blk_order.begin(), blk_order.end(), 0);
ie::SizeVector dim_offset(shape.size(), 0);
ie::SizeVector blk_strides;
auto byte_strides = element_type.bitwidth() >= 8 ? tensor_->get_strides() : Strides{};
if (byte_strides.empty()) {
blk_strides = ov::row_major_strides(shape);
} else {
blk_strides.resize(byte_strides.size());
std::transform(byte_strides.begin(),
byte_strides.end(),
blk_strides.begin(),
[&element_type](size_t byte_stride) {
OPENVINO_ASSERT(byte_stride % element_type.size() == 0,
"Limitation: Stride in bytes ",
byte_stride,
" should be divisible by size of element ",
element_type.size());
return byte_stride / element_type.size();
});
}
return ie::TensorDesc{ie::details::convertPrecision(element_type),
shape,
ie::BlockingDesc{shape, blk_order, 0, dim_offset, blk_strides}};
}(),
static_cast<T*>(tensor_->data()),
tensor_->get_byte_size()},
tensor{tensor_} {
explicit TensorMemoryBlob(const ov::SoPtr<ITensor>& tensor_, ie::TensorDesc desc) try : ie
::TBlob<T>{desc, static_cast<T*>(tensor_->data()), tensor_->get_byte_size()}, tensor{tensor_} {
OPENVINO_ASSERT(!std::dynamic_pointer_cast<ov::IRemoteTensor>(tensor._ptr));
}
catch (const std::exception& ex) {
Expand Down Expand Up @@ -620,7 +586,40 @@ const ie::Blob* get_hardware_blob(const ie::Blob* blob) {
return blob;
}

ie::Blob::Ptr tensor_to_blob(const ov::SoPtr<ITensor>& orig_tensor, bool unwrap) {
ie::Blob::Ptr tensor_to_blob(const ov::SoPtr<ITensor>& orig_tensor, bool unwrap, InferenceEngine::TensorDesc desc) {
auto create_desc = [](const ov::SoPtr<ov::ITensor>& tensor,
const InferenceEngine::TensorDesc& desc) -> InferenceEngine::TensorDesc {
if (desc.getLayout() != InferenceEngine::ANY ||
desc.getPrecision() != InferenceEngine::Precision::UNSPECIFIED) {
return desc;
}
auto element_type = tensor->get_element_type();
auto shape = tensor->get_shape();
ie::SizeVector blk_order(shape.size());
std::iota(blk_order.begin(), blk_order.end(), 0);
ie::SizeVector dim_offset(shape.size(), 0);
ie::SizeVector blk_strides;
auto byte_strides = element_type.bitwidth() >= 8 ? tensor->get_strides() : Strides{};
if (byte_strides.empty()) {
blk_strides = ov::row_major_strides(shape);
} else {
blk_strides.resize(byte_strides.size());
std::transform(byte_strides.begin(),
byte_strides.end(),
blk_strides.begin(),
[&element_type](size_t byte_stride) {
OPENVINO_ASSERT(byte_stride % element_type.size() == 0,
"Limitation: Stride in bytes ",
byte_stride,
" should be divisible by size of element ",
element_type.size());
return byte_stride / element_type.size();
});
}
return ie::TensorDesc{ie::details::convertPrecision(element_type),
shape,
ie::BlockingDesc{shape, blk_order, 0, dim_offset, blk_strides}};
};
#ifdef PROXY_PLUGIN_ENABLED
const auto& tensor = unwrap ? ov::proxy::get_hardware_tensor(orig_tensor) : orig_tensor;
#else
Expand All @@ -633,11 +632,11 @@ ie::Blob::Ptr tensor_to_blob(const ov::SoPtr<ITensor>& orig_tensor, bool unwrap)
} else if (auto blob_tensor = std::dynamic_pointer_cast<RemoteBlobTensor>(tensor._ptr)) {
return blob_tensor->blob;
} else if (std::dynamic_pointer_cast<ov::IRemoteTensor>(tensor._ptr)) {
return std::make_shared<TensorRemoteBlob>(tensor);
return std::make_shared<TensorRemoteBlob>(tensor, create_desc(tensor, desc));
} else {
#define CASE(precision, T) \
case element::precision: \
return std::make_shared<TensorMemoryBlob<T>>(tensor);
return std::make_shared<TensorMemoryBlob<T>>(tensor, create_desc(tensor, desc));
switch (tensor->get_element_type()) {
CASE(f32, float);
CASE(f64, double);
Expand All @@ -654,16 +653,16 @@ ie::Blob::Ptr tensor_to_blob(const ov::SoPtr<ITensor>& orig_tensor, bool unwrap)
CASE(u1, int8_t);
CASE(boolean, bool);
case element::f16:
return std::make_shared<TensorMemoryBlob<int16_t>>(tensor);
return std::make_shared<TensorMemoryBlob<int16_t>>(tensor, create_desc(tensor, desc));
case element::bf16:
return std::make_shared<TensorMemoryBlob<int16_t>>(tensor);
return std::make_shared<TensorMemoryBlob<int16_t>>(tensor, create_desc(tensor, desc));
default:
OPENVINO_THROW("Unsupported element type");
}
#undef CASE
}
OPENVINO_THROW("Cannot convert tensor to blob!");
}
} // namespace ov

namespace util {

Expand Down
19 changes: 18 additions & 1 deletion src/inference/src/dev/preprocessing/preprocessing.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,7 @@
#include "openvino/op/constant.hpp"
#include "openvino/pass/graph_rewrite.hpp"
#include "openvino/pass/manager.hpp"
#include "transformations/utils/utils.hpp"

bool ov::pass::AddPreprocessing::run_on_model(const std::shared_ptr<ov::Model>& model) {
RUN_ON_MODEL_SCOPE(AddPreprocessing);
Expand Down Expand Up @@ -111,8 +112,10 @@ bool ov::pass::AddPreprocessing::run_on_model(const std::shared_ptr<ov::Model>&
if (const_input.get_partial_shape().is_static() && const_input.get_shape().size() == 4)
preproc.input(i).model().set_layout("NCHW");
}
for (size_t i = 0; i < model->outputs().size(); i++) {
std::vector<std::string> legacy_names(model->get_output_size());
for (size_t i = 0; i < model->get_output_size(); i++) {
ov::Output<const Node> const_output(model->output(i).get_node(), model->output(i).get_index());
legacy_names[i] = ov::op::util::create_ie_output_name(const_output.get_node()->input_value(0));
InferenceEngine::DataPtr output_info;
// I don't remove rt info to have information in InputsInfo about pre-processing in legacy
// ExecutableNetwork
Expand All @@ -122,6 +125,15 @@ bool ov::pass::AddPreprocessing::run_on_model(const std::shared_ptr<ov::Model>&
if (element_type != model->output(i).get_element_type()) {
preproc.output(i).tensor().set_element_type(element_type);
}
if (output_info->getLayout() != InferenceEngine::Layout::BLOCKED &&
output_info->getLayout() != InferenceEngine::Layout::SCALAR) {
std::stringstream stream;
stream << output_info->getLayout();
preproc.output(i).tensor().set_layout(ov::Layout{stream.str()});
}

if (const_output.get_partial_shape().is_static() && const_output.get_shape().size() == 4)
preproc.output(i).model().set_layout("NCHW");
}

ov::pass::Manager manager(get_pass_config());
Expand All @@ -133,5 +145,10 @@ bool ov::pass::AddPreprocessing::run_on_model(const std::shared_ptr<ov::Model>&

preproc.build();

for (size_t i = 0; i < model->get_output_size(); i++) {
ov::descriptor::set_ov_tensor_legacy_name(model->output(i).get_node()->input_value(0).get_tensor(),
legacy_names[i]);
}

return false;
}
1 change: 1 addition & 0 deletions ...ugins/template/tests/functional/shared_tests_instances/behavior/plugin/set_preprocess.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -57,6 +57,7 @@ const std::vector<InferenceEngine::Layout> netLayouts = {

const std::vector<InferenceEngine::Layout> ioLayouts = {
InferenceEngine::Layout::NCHW,
InferenceEngine::Layout::NHWC,
};

INSTANTIATE_TEST_SUITE_P(smoke_BehaviorTests, InferRequestPreprocessConversionTest,
Expand Down

0 comments on commit 44cae12

Please sign in to comment.