Address feedback

triton-inference-server · Aug 10, 2023 · f33b82d · f33b82d
1 parent a704f59
commit f33b82d
Show file tree

Hide file tree

Showing 2 changed files with 28 additions and 9 deletions.
diff --git a/src/c++/perf_analyzer/inference_profiler.cc b/src/c++/perf_analyzer/inference_profiler.cc
@@ -1298,6 +1298,9 @@ InferenceProfiler::ValidLatencyMeasurement(
           (request_end_ns <= valid_range.second)) {
         valid_latencies->push_back(request_end_ns - request_start_ns);
         response_count += request_record.response_times_.size();
+        if (request_record.has_null_last_response_) {
+          response_count--;
+        }
         erase_indices.push_back(i);
         if (request_record.sequence_end_) {
           valid_sequence_count++;

diff --git a/src/c++/perf_analyzer/test_inference_profiler.cc b/src/c++/perf_analyzer/test_inference_profiler.cc
@@ -888,11 +888,27 @@ TEST_CASE(
     auto response3_timestamp{clock_epoch + std::chrono::nanoseconds(5)};
     auto response4_timestamp{clock_epoch + std::chrono::nanoseconds(6)};
     auto response5_timestamp{clock_epoch + std::chrono::nanoseconds(7)};
-    auto request_record2{RequestRecord(
-        request2_timestamp,
-        std::vector<std::chrono::time_point<std::chrono::system_clock>>{
-            response3_timestamp, response4_timestamp, response5_timestamp},
-        0, false, 0, false)};
+    RequestRecord request_record2{};
+    size_t expected_response_count{0};
+
+    SUBCASE("has three data responses")
+    {
+      request_record2 = RequestRecord(
+          request2_timestamp,
+          std::vector<std::chrono::time_point<std::chrono::system_clock>>{
+              response3_timestamp, response4_timestamp, response5_timestamp},
+          0, false, 0, false);
+      expected_response_count = 5;
+    }
+    SUBCASE("has two data responses and one null response")
+    {
+      request_record2 = RequestRecord(
+          request2_timestamp,
+          std::vector<std::chrono::time_point<std::chrono::system_clock>>{
+              response3_timestamp, response4_timestamp, response5_timestamp},
+          0, false, 0, true);
+      expected_response_count = 4;
+    }
 
     mock_inference_profiler.all_request_records_ = {
         request_record1, request_record2};
@@ -909,7 +925,7 @@ TEST_CASE(
         valid_range, valid_sequence_count, delayed_request_count,
         &valid_latencies, response_count, valid_requests);
 
-    CHECK(response_count == 5);
+    CHECK(response_count == expected_response_count);
   }
   SUBCASE("testing logic relevant to valid request output")
   {
@@ -921,23 +937,23 @@ TEST_CASE(
         request1_timestamp,
         std::vector<std::chrono::time_point<std::chrono::system_clock>>{
             response1_timestamp},
-        0, false, 0)};
+        0, false, 0, false)};
 
     auto request2_timestamp{clock_epoch + std::chrono::nanoseconds(3)};
     auto response2_timestamp{clock_epoch + std::chrono::nanoseconds(4)};
     auto request_record2{RequestRecord(
         request2_timestamp,
         std::vector<std::chrono::time_point<std::chrono::system_clock>>{
             response2_timestamp},
-        0, false, 0)};
+        0, false, 0, false)};
 
     auto request3_timestamp{clock_epoch + std::chrono::nanoseconds(5)};
     auto response3_timestamp{clock_epoch + std::chrono::nanoseconds(6)};
     auto request_record3{RequestRecord(
         request3_timestamp,
         std::vector<std::chrono::time_point<std::chrono::system_clock>>{
             response3_timestamp},
-        0, false, 0)};
+        0, false, 0, false)};
 
     mock_inference_profiler.all_request_records_ = {
         request_record1, request_record2, request_record3};