From 2f7b455c85e3d8a57dfb2b7df352029052622ee6 Mon Sep 17 00:00:00 2001 From: Matthew Kotila Date: Tue, 8 Aug 2023 23:29:18 +0000 Subject: [PATCH] Address feedback --- src/c++/perf_analyzer/CMakeLists.txt | 8 +- .../client_backend/client_backend.h | 2 + src/c++/perf_analyzer/inference_profiler.cc | 6 +- src/c++/perf_analyzer/inference_profiler.h | 13 ++- src/c++/perf_analyzer/perf_analyzer.cc | 17 ++-- src/c++/perf_analyzer/perf_analyzer.h | 12 +-- ...collector.cc => profile_data_collector.cc} | 19 ++-- ...a_collector.h => profile_data_collector.h} | 35 ++++--- ...a_reporter.cc => profile_data_exporter.cc} | 99 ++++++++++--------- ...ata_reporter.h => profile_data_exporter.h} | 29 +++--- 10 files changed, 129 insertions(+), 111 deletions(-) rename src/c++/perf_analyzer/{raw_data_collector.cc => profile_data_collector.cc} (84%) rename src/c++/perf_analyzer/{raw_data_collector.h => profile_data_collector.h} (74%) rename src/c++/perf_analyzer/{raw_data_reporter.cc => profile_data_exporter.cc} (67%) rename src/c++/perf_analyzer/{raw_data_reporter.h => profile_data_exporter.h} (78%) diff --git a/src/c++/perf_analyzer/CMakeLists.txt b/src/c++/perf_analyzer/CMakeLists.txt index 5cd8b8dd5..ea4065c7e 100644 --- a/src/c++/perf_analyzer/CMakeLists.txt +++ b/src/c++/perf_analyzer/CMakeLists.txt @@ -67,8 +67,8 @@ set( infer_data_manager.cc infer_data_manager_shm.cc sequence_manager.cc - raw_data_collector.cc - raw_data_reporter.cc + profile_data_collector.cc + profile_data_exporter.cc ) set( @@ -107,8 +107,8 @@ set( fifo_ctx_id_tracker.h rand_ctx_id_tracker.h request_record.h - raw_data_collector.h - raw_data_reporter.h + profile_data_collector.h + profile_data_exporter.h ) add_executable( diff --git a/src/c++/perf_analyzer/client_backend/client_backend.h b/src/c++/perf_analyzer/client_backend/client_backend.h index c351dfe61..dd632e67e 100644 --- a/src/c++/perf_analyzer/client_backend/client_backend.h +++ b/src/c++/perf_analyzer/client_backend/client_backend.h @@ -633,3 +633,5 @@ class InferResult { }; }}} // namespace triton::perfanalyzer::clientbackend + +namespace cb = triton::perfanalyzer::clientbackend; diff --git a/src/c++/perf_analyzer/inference_profiler.cc b/src/c++/perf_analyzer/inference_profiler.cc index 64211946a..849499bbf 100644 --- a/src/c++/perf_analyzer/inference_profiler.cc +++ b/src/c++/perf_analyzer/inference_profiler.cc @@ -471,7 +471,7 @@ InferenceProfiler::Create( uint64_t measurement_request_count, MeasurementMode measurement_mode, std::shared_ptr mpi_driver, const uint64_t metrics_interval_ms, const bool should_collect_metrics, const double overhead_pct_threshold, - const std::shared_ptr collector) + const std::shared_ptr collector) { std::unique_ptr local_profiler(new InferenceProfiler( verbose, stability_threshold, measurement_window_ms, max_trials, @@ -495,7 +495,7 @@ InferenceProfiler::InferenceProfiler( MeasurementMode measurement_mode, std::shared_ptr mpi_driver, const uint64_t metrics_interval_ms, const bool should_collect_metrics, const double overhead_pct_threshold, - const std::shared_ptr collector) + const std::shared_ptr collector) : verbose_(verbose), measurement_window_ms_(measurement_window_ms), max_trials_(max_trials), extra_percentile_(extra_percentile), percentile_(percentile), latency_threshold_ms_(latency_threshold_ms_), @@ -1316,7 +1316,7 @@ InferenceProfiler::CollectData( PerfStatus& summary, uint64_t window_start_ns, uint64_t window_end_ns, std::vector&& request_records) { - PerfMode id{summary.concurrency, summary.request_rate}; + InferenceLoadMode id{summary.concurrency, summary.request_rate}; collector_->AddWindow(id, window_start_ns, window_end_ns); collector_->AddData(id, std::move(request_records)); } diff --git a/src/c++/perf_analyzer/inference_profiler.h b/src/c++/perf_analyzer/inference_profiler.h index 0fb0bcd40..0af9c8af2 100644 --- a/src/c++/perf_analyzer/inference_profiler.h +++ b/src/c++/perf_analyzer/inference_profiler.h @@ -43,7 +43,7 @@ #include "metrics_manager.h" #include "model_parser.h" #include "mpi_utils.h" -#include "raw_data_collector.h" +#include "profile_data_collector.h" #include "request_rate_manager.h" namespace triton { namespace perfanalyzer { @@ -222,8 +222,7 @@ class InferenceProfiler { /// should be collected. /// \param overhead_pct_threshold User set threshold above which the PA /// overhead is too significant to provide usable results. - /// \param collector Collects all of the raw data from experiments into a - /// central location + /// \param collector Collector for the profile data from experiments /// \return cb::Error object indicating success or failure. static cb::Error Create( const bool verbose, const double stability_threshold, @@ -236,7 +235,7 @@ class InferenceProfiler { uint64_t measurement_request_count, MeasurementMode measurement_mode, std::shared_ptr mpi_driver, const uint64_t metrics_interval_ms, const bool should_collect_metrics, const double overhead_pct_threshold, - const std::shared_ptr collector); + const std::shared_ptr collector); /// Performs the profiling on the given range with the given search algorithm. /// For profiling using request rate invoke template with double, otherwise @@ -319,7 +318,7 @@ class InferenceProfiler { MeasurementMode measurement_mode, std::shared_ptr mpi_driver, const uint64_t metrics_interval_ms, const bool should_collect_metrics, const double overhead_pct_threshold, - const std::shared_ptr collector); + const std::shared_ptr collector); /// Actively measure throughput in every 'measurement_window' msec until the /// throughput is stable. Once the throughput is stable, it adds the @@ -458,7 +457,7 @@ class InferenceProfiler { std::vector& valid_requests); /// Add the data from the request records to the Raw Data Collector - /// \param perf_status Perf_status of the current measurement + /// \param perf_status PerfStatus of the current measurement /// \param window_start_ns The window start timestamp in nanoseconds. /// \param window_end_ns The window end timestamp in nanoseconds. /// \param request_records The request records to collect. @@ -688,7 +687,7 @@ class InferenceProfiler { std::shared_ptr parser_; std::shared_ptr profile_backend_; std::unique_ptr manager_; - std::shared_ptr collector_; + std::shared_ptr collector_; LoadParams load_parameters_; bool include_lib_stats_; diff --git a/src/c++/perf_analyzer/perf_analyzer.cc b/src/c++/perf_analyzer/perf_analyzer.cc index 836c5171a..29285bc2f 100644 --- a/src/c++/perf_analyzer/perf_analyzer.cc +++ b/src/c++/perf_analyzer/perf_analyzer.cc @@ -64,7 +64,7 @@ PerfAnalyzer::Run() PrerunReport(); Profile(); WriteReport(); - GenerateProfileExportReport(); + GenerateProfileExport(); Finalize(); } @@ -255,12 +255,12 @@ PerfAnalyzer::CreateAnalyzerObjects() params_->sequence_length_specified, params_->sequence_length_variation); FAIL_IF_ERR( - pa::RawDataCollector::Create(&collector_), - "failed to create data collector"); + pa::ProfileDataCollector::Create(&collector_), + "failed to create profile data collector"); FAIL_IF_ERR( - pa::RawDataReporter::Create(&reporter_), - "failed to create data reporter"); + pa::ProfileDataExporter::Create(&exporter_), + "failed to create profile data exporter"); FAIL_IF_ERR( pa::InferenceProfiler::Create( @@ -432,11 +432,12 @@ PerfAnalyzer::WriteReport() } void -PerfAnalyzer::GenerateProfileExportReport() +PerfAnalyzer::GenerateProfileExport() { if (!params_->profile_export_file.empty()) { - reporter_->ConvertToJson(collector_->GetData(), collector_->GetVersion()); - reporter_->OutputToFile(params_->profile_export_file); + exporter_->Export( + collector_->GetData(), collector_->GetVersion(), + params_->profile_export_file); } } diff --git a/src/c++/perf_analyzer/perf_analyzer.h b/src/c++/perf_analyzer/perf_analyzer.h index 29ccef409..b75fe35f0 100644 --- a/src/c++/perf_analyzer/perf_analyzer.h +++ b/src/c++/perf_analyzer/perf_analyzer.h @@ -1,4 +1,4 @@ -// Copyright 2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// Copyright 2022-2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. // // Redistribution and use in source and binary forms, with or without // modification, are permitted provided that the following conditions @@ -37,8 +37,8 @@ #include "model_parser.h" #include "mpi_utils.h" #include "perf_utils.h" -#include "raw_data_collector.h" -#include "raw_data_reporter.h" +#include "profile_data_collector.h" +#include "profile_data_exporter.h" // Perf Analyzer provides various metrics to measure the performance of // the inference server. It can either be used to measure the throughput, @@ -184,8 +184,8 @@ class PerfAnalyzer { std::unique_ptr backend_; std::shared_ptr parser_; std::vector perf_statuses_; - std::shared_ptr collector_; - std::shared_ptr reporter_; + std::shared_ptr collector_; + std::shared_ptr exporter_; // // Helper methods @@ -197,6 +197,6 @@ class PerfAnalyzer { void PrerunReport(); void Profile(); void WriteReport(); - void GenerateProfileExportReport(); + void GenerateProfileExport(); void Finalize(); }; diff --git a/src/c++/perf_analyzer/raw_data_collector.cc b/src/c++/perf_analyzer/profile_data_collector.cc similarity index 84% rename from src/c++/perf_analyzer/raw_data_collector.cc rename to src/c++/perf_analyzer/profile_data_collector.cc index 483927286..4a27a7767 100644 --- a/src/c++/perf_analyzer/raw_data_collector.cc +++ b/src/c++/perf_analyzer/profile_data_collector.cc @@ -24,7 +24,7 @@ // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -#include "raw_data_collector.h" +#include "profile_data_collector.h" #include @@ -33,18 +33,19 @@ namespace triton { namespace perfanalyzer { cb::Error -RawDataCollector::Create(std::shared_ptr* collector) +ProfileDataCollector::Create(std::shared_ptr* collector) { - std::shared_ptr local_collector{new RawDataCollector()}; + std::shared_ptr local_collector{ + new ProfileDataCollector()}; *collector = std::move(local_collector); return cb::Error::Success; } void -RawDataCollector::AddWindow( - PerfMode& id, uint64_t window_start_ns, uint64_t window_end_ns) +ProfileDataCollector::AddWindow( + InferenceLoadMode& id, uint64_t window_start_ns, uint64_t window_end_ns) { - auto it = FindEntry(id); + auto it = FindExperiment(id); if (it == experiments_.end()) { Experiment new_experiment{}; @@ -64,10 +65,10 @@ RawDataCollector::AddWindow( } void -RawDataCollector::AddData( - PerfMode& id, std::vector&& request_records) +ProfileDataCollector::AddData( + InferenceLoadMode& id, std::vector&& request_records) { - auto it = FindEntry(id); + auto it = FindExperiment(id); if (it == experiments_.end()) { Experiment new_experiment{}; diff --git a/src/c++/perf_analyzer/raw_data_collector.h b/src/c++/perf_analyzer/profile_data_collector.h similarity index 74% rename from src/c++/perf_analyzer/raw_data_collector.h rename to src/c++/perf_analyzer/profile_data_collector.h index 2b48a4f27..b423b7317 100644 --- a/src/c++/perf_analyzer/raw_data_collector.h +++ b/src/c++/perf_analyzer/profile_data_collector.h @@ -37,39 +37,45 @@ namespace triton { namespace perfanalyzer { -struct PerfMode { +/// Data structure to hold which inference load mode was used for an experiment. +/// Only one data member will be nonzero, indicating the inference load mode for +/// a particular experiment. +struct InferenceLoadMode { uint32_t concurrency; double request_rate; - PerfMode() + InferenceLoadMode() { concurrency = 0; request_rate = 0.0; } - PerfMode(uint64_t c, double rr) + InferenceLoadMode(uint64_t c, double rr) { concurrency = c; request_rate = rr; } - bool operator==(const PerfMode& rhs) const + bool operator==(const InferenceLoadMode& rhs) const { return (concurrency == rhs.concurrency) && (request_rate == rhs.request_rate); } }; +/// Data structure to hold profile export data for an experiment (e.g. +/// concurrency 4 or request rate 50) struct Experiment { - PerfMode mode; + InferenceLoadMode mode; std::vector requests; std::vector window_boundaries; }; -class RawDataCollector { +/// Data structure and methods for storing profile export data. +class ProfileDataCollector { public: - static cb::Error Create(std::shared_ptr* collector); - ~RawDataCollector() = default; + static cb::Error Create(std::shared_ptr* collector); + ~ProfileDataCollector() = default; /// Add a measurement window to the collector @@ -77,23 +83,24 @@ class RawDataCollector { /// \param window_start_ns The window start timestamp in nanoseconds. /// \param window_end_ns The window end timestamp in nanoseconds. void AddWindow( - PerfMode& id, uint64_t window_start_ns, uint64_t window_end_ns); + InferenceLoadMode& id, uint64_t window_start_ns, uint64_t window_end_ns); /// Add request records to an experiment /// @param id Identifier for the experiment /// @param request_records The request information for the current experiment. - void AddData(PerfMode& id, std::vector&& request_records); + void AddData( + InferenceLoadMode& id, std::vector&& request_records); - /// Get the raw experiment data for the profile - /// @return Raw experiment data + /// Get the experiment data for the profile + /// @return Experiment data std::vector& GetData() { return experiments_; } std::string& GetVersion() { return version_; } private: - RawDataCollector() = default; + ProfileDataCollector() = default; - std::vector::iterator FindEntry(PerfMode& id) + std::vector::iterator FindExperiment(InferenceLoadMode& id) { return std::find_if( experiments_.begin(), experiments_.end(), diff --git a/src/c++/perf_analyzer/raw_data_reporter.cc b/src/c++/perf_analyzer/profile_data_exporter.cc similarity index 67% rename from src/c++/perf_analyzer/raw_data_reporter.cc rename to src/c++/perf_analyzer/profile_data_exporter.cc index e60520f3c..7c5fffbaa 100644 --- a/src/c++/perf_analyzer/raw_data_reporter.cc +++ b/src/c++/perf_analyzer/profile_data_exporter.cc @@ -25,7 +25,7 @@ // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -#include "raw_data_reporter.h" +#include "profile_data_exporter.h" #include #include @@ -35,28 +35,36 @@ namespace triton { namespace perfanalyzer { -using namespace rapidjson; - cb::Error -RawDataReporter::Create(std::shared_ptr* reporter) +ProfileDataExporter::Create(std::shared_ptr* exporter) { - std::shared_ptr local_reporter{new RawDataReporter()}; - *reporter = std::move(local_reporter); + std::shared_ptr local_exporter{ + new ProfileDataExporter()}; + *exporter = std::move(local_exporter); return cb::Error::Success; } void -RawDataReporter::ConvertToJson( - std::vector& raw_experiments, std::string& raw_version) +ProfileDataExporter::Export( + const std::vector& raw_experiments, std::string& raw_version, + std::string& file_path) +{ + ConvertToJson(raw_experiments, raw_version); + OutputToFile(file_path); +} + +void +ProfileDataExporter::ConvertToJson( + const std::vector& raw_experiments, std::string& raw_version) { ClearDocument(); - Value experiments(kArrayType); + rapidjson::Value experiments(rapidjson::kArrayType); for (const auto& raw_experiment : raw_experiments) { - Value entry(kObjectType); - Value experiment(kObjectType); - Value requests(kArrayType); - Value window_boundaries(kArrayType); + rapidjson::Value entry(rapidjson::kObjectType); + rapidjson::Value experiment(rapidjson::kObjectType); + rapidjson::Value requests(rapidjson::kArrayType); + rapidjson::Value window_boundaries(rapidjson::kArrayType); AddExperiment(entry, experiment, raw_experiment); AddRequests(entry, requests, raw_experiment); @@ -70,24 +78,25 @@ RawDataReporter::ConvertToJson( } void -RawDataReporter::ClearDocument() +ProfileDataExporter::ClearDocument() { - Document d{}; + rapidjson::Document d{}; document_.Swap(d); document_.SetObject(); } void -RawDataReporter::AddExperiment( - Value& entry, Value& experiment, const Experiment& raw_experiment) +ProfileDataExporter::AddExperiment( + rapidjson::Value& entry, rapidjson::Value& experiment, + const Experiment& raw_experiment) { - Value mode; - Value value; + rapidjson::Value mode; + rapidjson::Value value; if (raw_experiment.mode.concurrency != 0) { - mode = StringRef("concurrency"); + mode = rapidjson::StringRef("concurrency"); value.SetUint64(raw_experiment.mode.concurrency); } else { - mode = StringRef("request_rate"); + mode = rapidjson::StringRef("request_rate"); value.SetUint64(raw_experiment.mode.request_rate); } experiment.AddMember("mode", mode, document_.GetAllocator()); @@ -96,23 +105,24 @@ RawDataReporter::AddExperiment( } void -RawDataReporter::AddRequests( - Value& entry, Value& requests, const Experiment& raw_experiment) +ProfileDataExporter::AddRequests( + rapidjson::Value& entry, rapidjson::Value& requests, + const Experiment& raw_experiment) { for (auto& raw_request : raw_experiment.requests) { - Value request(kObjectType); - Value timestamp; + rapidjson::Value request(rapidjson::kObjectType); + rapidjson::Value timestamp; timestamp.SetUint64(raw_request.start_time_.time_since_epoch().count()); request.AddMember("timestamp", timestamp, document_.GetAllocator()); if (raw_request.sequence_id_ != 0) { - Value sequence_id; + rapidjson::Value sequence_id; sequence_id.SetUint64(raw_request.sequence_id_); request.AddMember("sequence_id", sequence_id, document_.GetAllocator()); } - Value responses(kArrayType); + rapidjson::Value responses(rapidjson::kArrayType); AddResponses(responses, raw_request.response_times_); request.AddMember( "response_timestamps", responses, document_.GetAllocator()); @@ -122,24 +132,25 @@ RawDataReporter::AddRequests( } void -RawDataReporter::AddResponses( - Value& responses, +ProfileDataExporter::AddResponses( + rapidjson::Value& responses, const std::vector>& response_times) { for (auto& response : response_times) { - Value time; + rapidjson::Value time; time.SetUint64(response.time_since_epoch().count()); responses.PushBack(time, document_.GetAllocator()); } } void -RawDataReporter::AddWindowBoundaries( - Value& entry, Value& window_boundaries, const Experiment& raw_experiment) +ProfileDataExporter::AddWindowBoundaries( + rapidjson::Value& entry, rapidjson::Value& window_boundaries, + const Experiment& raw_experiment) { for (auto& window : raw_experiment.window_boundaries) { - Value w; + rapidjson::Value w; w.SetUint64(window); window_boundaries.PushBack(w, document_.GetAllocator()); } @@ -148,25 +159,15 @@ RawDataReporter::AddWindowBoundaries( } void -RawDataReporter::AddVersion(std::string& raw_version) +ProfileDataExporter::AddVersion(std::string& raw_version) { - Value version; - version = StringRef(raw_version.c_str()); + rapidjson::Value version; + version = rapidjson::StringRef(raw_version.c_str()); document_.AddMember("version", version, document_.GetAllocator()); } -// print to file - -void -RawDataReporter::Print() -{ - OStreamWrapper out(std::cout); - Writer writer(out); - document_.Accept(writer); -} - void -RawDataReporter::OutputToFile(std::string& file_path) +ProfileDataExporter::OutputToFile(std::string& file_path) { FILE* fp = fopen(file_path.c_str(), "w"); if (fp == nullptr) { @@ -174,9 +175,9 @@ RawDataReporter::OutputToFile(std::string& file_path) "failed to open file for outputting raw profile data", GENERIC_ERROR); } char writeBuffer[65536]; - FileWriteStream os(fp, writeBuffer, sizeof(writeBuffer)); + rapidjson::FileWriteStream os(fp, writeBuffer, sizeof(writeBuffer)); - Writer writer(os); + rapidjson::Writer writer(os); document_.Accept(writer); fclose(fp); diff --git a/src/c++/perf_analyzer/raw_data_reporter.h b/src/c++/perf_analyzer/profile_data_exporter.h similarity index 78% rename from src/c++/perf_analyzer/raw_data_reporter.h rename to src/c++/perf_analyzer/profile_data_exporter.h index 9f268b80c..361e11fa0 100644 --- a/src/c++/perf_analyzer/raw_data_reporter.h +++ b/src/c++/perf_analyzer/profile_data_exporter.h @@ -29,31 +29,38 @@ #include #include "client_backend/client_backend.h" -#include "raw_data_collector.h" +#include "profile_data_collector.h" namespace triton { namespace perfanalyzer { -class RawDataReporter { +/// Exports profile data. +class ProfileDataExporter { public: - static cb::Error Create(std::shared_ptr* reporter); - ~RawDataReporter() = default; + static cb::Error Create(std::shared_ptr* exporter); + ~ProfileDataExporter() = default; + /// Export profile data to json file + /// @param raw_experiments All of the raw data for the experiments run by perf + /// analyzer + /// @param raw_version String containing the version number for the json + /// output + /// @param file_path File path to export profile data to. + void Export( + const std::vector& raw_experiments, std::string& raw_version, + std::string& file_path); + + private: + ProfileDataExporter() = default; /// Convert the raw data collected to json output /// @param raw_experiments All of the raw data for the experiments run by perf /// analyzer /// @param raw_version String containing the version number for the json /// output void ConvertToJson( - std::vector& raw_experiments, std::string& raw_version); - - /// Output to stdout - void Print(); + const std::vector& raw_experiments, std::string& raw_version); void OutputToFile(std::string& file_path); - - private: - RawDataReporter() = default; void ClearDocument(); void AddExperiment( rapidjson::Value& entry, rapidjson::Value& experiment,