From 0d1bcf1a84946424fffe9a785422bb78701eb78e Mon Sep 17 00:00:00 2001 From: jinhelin Date: Mon, 18 Mar 2024 13:33:42 +0800 Subject: [PATCH] This is an automated cherry-pick of #51726 Signed-off-by: ti-chi-bot --- DEPS.bzl | 10 + go.mod | 8 + go.sum | 11 + pkg/util/execdetails/execdetails.go | 295 +++++++++++++++++++---- pkg/util/execdetails/execdetails_test.go | 38 ++- 5 files changed, 290 insertions(+), 72 deletions(-) diff --git a/DEPS.bzl b/DEPS.bzl index 0010bd6297fe3..1c49b8e5a51ae 100644 --- a/DEPS.bzl +++ b/DEPS.bzl @@ -5858,6 +5858,7 @@ def go_deps(): name = "com_github_pingcap_tipb", build_file_proto_mode = "disable_global", importpath = "github.com/pingcap/tipb", +<<<<<<< HEAD sha256 = "95f1bd8447c012c516789e30c09f72149f14e9df53d58bc3f74d54b7473119a0", strip_prefix = "github.com/pingcap/tipb@v0.0.0-20230919054518-dfd7d194838f", urls = [ @@ -5865,6 +5866,15 @@ def go_deps(): "http://ats.apps.svc/gomod/github.com/pingcap/tipb/com_github_pingcap_tipb-v0.0.0-20230919054518-dfd7d194838f.zip", "https://cache.hawkingrei.com/gomod/github.com/pingcap/tipb/com_github_pingcap_tipb-v0.0.0-20230919054518-dfd7d194838f.zip", "https://storage.googleapis.com/pingcapmirror/gomod/github.com/pingcap/tipb/com_github_pingcap_tipb-v0.0.0-20230919054518-dfd7d194838f.zip", +======= + sha256 = "42c365f3f99d2577fe29c89d433894b2d0d69b054248bb3bafbced33332933e3", + strip_prefix = "github.com/pingcap/tipb@v0.0.0-20240318032315-55a7867ddd50", + urls = [ + "http://bazel-cache.pingcap.net:8080/gomod/github.com/pingcap/tipb/com_github_pingcap_tipb-v0.0.0-20240318032315-55a7867ddd50.zip", + "http://ats.apps.svc/gomod/github.com/pingcap/tipb/com_github_pingcap_tipb-v0.0.0-20240318032315-55a7867ddd50.zip", + "https://cache.hawkingrei.com/gomod/github.com/pingcap/tipb/com_github_pingcap_tipb-v0.0.0-20240318032315-55a7867ddd50.zip", + "https://storage.googleapis.com/pingcapmirror/gomod/github.com/pingcap/tipb/com_github_pingcap_tipb-v0.0.0-20240318032315-55a7867ddd50.zip", +>>>>>>> 3f915c0a369 (execdetails: add statistical data of table scanning of TiFlash (#51726)) ], ) go_repository( diff --git a/go.mod b/go.mod index 71a42a67d9d09..740e4f455e595 100644 --- a/go.mod +++ b/go.mod @@ -86,12 +86,20 @@ require ( github.com/pingcap/log v1.1.1-0.20230317032135-a0d097d16e22 github.com/pingcap/sysutil v1.0.1-0.20230407040306-fb007c5aff21 github.com/pingcap/tidb/pkg/parser v0.0.0-20211011031125-9b13dc409c5e +<<<<<<< HEAD github.com/pingcap/tipb v0.0.0-20230919054518-dfd7d194838f github.com/pkg/errors v0.9.1 github.com/prometheus/client_golang v1.18.0 github.com/prometheus/client_model v0.5.0 github.com/prometheus/common v0.46.0 github.com/prometheus/prometheus v0.0.0-20190525122359-d20e84d0fb64 +======= + github.com/pingcap/tipb v0.0.0-20240318032315-55a7867ddd50 + github.com/prometheus/client_golang v1.19.0 + github.com/prometheus/client_model v0.6.0 + github.com/prometheus/common v0.48.0 + github.com/prometheus/prometheus v0.50.1 +>>>>>>> 3f915c0a369 (execdetails: add statistical data of table scanning of TiFlash (#51726)) github.com/robfig/cron/v3 v3.0.1 github.com/sasha-s/go-deadlock v0.2.0 github.com/shirou/gopsutil/v3 v3.23.5 diff --git a/go.sum b/go.sum index 18753f45771d4..acccd59db1c7b 100644 --- a/go.sum +++ b/go.sum @@ -815,6 +815,7 @@ github.com/pingcap/kvproto v0.0.0-20240208102409-a554af8ee11f h1:2xvTjl4OrQY+XF3 github.com/pingcap/kvproto v0.0.0-20240208102409-a554af8ee11f/go.mod h1:rXxWk2UnwfUhLXha1jxRWPADw9eMZGWEWCg92Tgmb/8= github.com/pingcap/log v0.0.0-20210625125904-98ed8e2eb1c7/go.mod h1:8AanEdAHATuRurdGxZXBz0At+9avep+ub7U1AGYLIMM= github.com/pingcap/log v1.1.0/go.mod h1:DWQW5jICDR7UJh4HtxXSM20Churx4CQL0fwL/SoOSA4= +<<<<<<< HEAD github.com/pingcap/log v1.1.1-0.20230317032135-a0d097d16e22 h1:2SOzvGvE8beiC1Y4g9Onkvu6UmuBBOeWRGQEjJaT/JY= github.com/pingcap/log v1.1.1-0.20230317032135-a0d097d16e22/go.mod h1:DWQW5jICDR7UJh4HtxXSM20Churx4CQL0fwL/SoOSA4= github.com/pingcap/sysutil v1.0.1-0.20230407040306-fb007c5aff21 h1:QV6jqlfOkh8hqvEAgwBZa+4bSgO0EeKC7s5c6Luam2I= @@ -823,6 +824,16 @@ github.com/pingcap/tipb v0.0.0-20230919054518-dfd7d194838f h1:NCiI4Wyu4GkViLGTu6 github.com/pingcap/tipb v0.0.0-20230919054518-dfd7d194838f/go.mod h1:A7mrd7WHBl1o63LE2bIBGEJMTNWXqhgmYiOvMLxozfs= github.com/pkg/browser v0.0.0-20210115035449-ce105d075bb4 h1:Qj1ukM4GlMWXNdMBuXcXfz/Kw9s1qm0CLY32QxuSImI= github.com/pkg/browser v0.0.0-20210115035449-ce105d075bb4/go.mod h1:N6UoU20jOqggOuDwUaBQpluzLNDqif3kq9z2wpdYEfQ= +======= +github.com/pingcap/log v1.1.1-0.20240314023424-862ccc32f18d h1:y3EueKVfVykdpTyfUnQGqft0ud+xVFuCdp1XkVL0X1E= +github.com/pingcap/log v1.1.1-0.20240314023424-862ccc32f18d/go.mod h1:ORfBOFp1eteu2odzsyaxI+b8TzJwgjwyQcGhI+9SfEA= +github.com/pingcap/sysutil v1.0.1-0.20240311050922-ae81ee01f3a5 h1:T4pXRhBflzDeAhmOQHNPRRogMYxP13V7BkYw3ZsoSfE= +github.com/pingcap/sysutil v1.0.1-0.20240311050922-ae81ee01f3a5/go.mod h1:rlimy0GcTvjiJqvD5mXTRr8O2eNZPBrcUgiWVYp9530= +github.com/pingcap/tipb v0.0.0-20240318032315-55a7867ddd50 h1:fVNBE06Rjec+EIHaYAKAHa/bIt5lnu3Zh9O6kV7ZAdg= +github.com/pingcap/tipb v0.0.0-20240318032315-55a7867ddd50/go.mod h1:A7mrd7WHBl1o63LE2bIBGEJMTNWXqhgmYiOvMLxozfs= +github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c h1:+mdjkGKdHQG3305AYmdv1U2eRNDiU2ErMBj1gwrq8eQ= +github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c/go.mod h1:7rwL4CYBLnjLxUqIJNnCWiEdr3bn6IUYi15bNlnbCCU= +>>>>>>> 3f915c0a369 (execdetails: add statistical data of table scanning of TiFlash (#51726)) github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA= github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= diff --git a/pkg/util/execdetails/execdetails.go b/pkg/util/execdetails/execdetails.go index a76bb18725729..1a546e32dd57c 100644 --- a/pkg/util/execdetails/execdetails.go +++ b/pkg/util/execdetails/execdetails.go @@ -583,20 +583,42 @@ func (crs *CopRuntimeStats) RecordOneCopTask(address string, summary *tipb.Execu storeType: crs.storeType, BasicRuntimeStats: BasicRuntimeStats{ tiflashScanContext: TiFlashScanContext{ - totalDmfileScannedPacks: summary.GetTiflashScanContext().GetTotalDmfileScannedPacks(), - totalDmfileSkippedPacks: summary.GetTiflashScanContext().GetTotalDmfileSkippedPacks(), - totalDmfileScannedRows: summary.GetTiflashScanContext().GetTotalDmfileScannedRows(), - totalDmfileSkippedRows: summary.GetTiflashScanContext().GetTotalDmfileSkippedRows(), - totalDmfileRoughSetIndexCheckTimeMs: summary.GetTiflashScanContext().GetTotalDmfileRoughSetIndexCheckTimeMs(), - totalDmfileReadTimeMs: summary.GetTiflashScanContext().GetTotalDmfileReadTimeMs(), - totalCreateSnapshotTimeMs: summary.GetTiflashScanContext().GetTotalCreateSnapshotTimeMs(), - totalLocalRegionNum: summary.GetTiflashScanContext().GetTotalLocalRegionNum(), - totalRemoteRegionNum: summary.GetTiflashScanContext().GetTotalRemoteRegionNum(), - totalLearnerReadMs: summary.GetTiflashScanContext().GetTotalLearnerReadMs(), - totalDisaggReadCacheHitSize: summary.GetTiflashScanContext().GetTotalDisaggReadCacheHitSize(), - totalDisaggReadCacheMissSize: summary.GetTiflashScanContext().GetTotalDisaggReadCacheMissSize()}}, threads: int32(summary.GetConcurrency()), + dmfileDataScannedRows: summary.GetTiflashScanContext().GetDmfileDataScannedRows(), + dmfileDataSkippedRows: summary.GetTiflashScanContext().GetDmfileDataSkippedRows(), + dmfileMvccScannedRows: summary.GetTiflashScanContext().GetDmfileMvccScannedRows(), + dmfileMvccSkippedRows: summary.GetTiflashScanContext().GetDmfileMvccSkippedRows(), + dmfileLmFilterScannedRows: summary.GetTiflashScanContext().GetDmfileLmFilterScannedRows(), + dmfileLmFilterSkippedRows: summary.GetTiflashScanContext().GetDmfileLmFilterSkippedRows(), + totalDmfileRsCheckMs: summary.GetTiflashScanContext().GetTotalDmfileRsCheckMs(), + totalDmfileReadMs: summary.GetTiflashScanContext().GetTotalDmfileReadMs(), + totalBuildSnapshotMs: summary.GetTiflashScanContext().GetTotalBuildSnapshotMs(), + localRegions: summary.GetTiflashScanContext().GetLocalRegions(), + remoteRegions: summary.GetTiflashScanContext().GetRemoteRegions(), + totalLearnerReadMs: summary.GetTiflashScanContext().GetTotalLearnerReadMs(), + disaggReadCacheHitBytes: summary.GetTiflashScanContext().GetDisaggReadCacheHitBytes(), + disaggReadCacheMissBytes: summary.GetTiflashScanContext().GetDisaggReadCacheMissBytes(), + segments: summary.GetTiflashScanContext().GetSegments(), + readTasks: summary.GetTiflashScanContext().GetReadTasks(), + deltaRows: summary.GetTiflashScanContext().GetDeltaRows(), + deltaBytes: summary.GetTiflashScanContext().GetDeltaBytes(), + mvccInputRows: summary.GetTiflashScanContext().GetMvccInputRows(), + mvccInputBytes: summary.GetTiflashScanContext().GetMvccInputBytes(), + mvccOutputRows: summary.GetTiflashScanContext().GetMvccOutputRows(), + lmSkipRows: summary.GetTiflashScanContext().GetLmSkipRows(), + totalBuildBitmapMs: summary.GetTiflashScanContext().GetTotalBuildBitmapMs(), + totalBuildInputStreamMs: summary.GetTiflashScanContext().GetTotalBuildInputstreamMs(), + staleReadRegions: summary.GetTiflashScanContext().GetStaleReadRegions(), + minLocalStreamMs: summary.GetTiflashScanContext().GetMinLocalStreamMs(), + maxLocalStreamMs: summary.GetTiflashScanContext().GetMaxLocalStreamMs(), + minRemoteStreamMs: summary.GetTiflashScanContext().GetMinRemoteStreamMs(), + maxRemoteStreamMs: summary.GetTiflashScanContext().GetMaxLocalStreamMs(), + regionsOfInstance: make(map[string]uint64), + }}, threads: int32(summary.GetConcurrency()), totalTasks: 1, } + for _, instance := range summary.GetTiflashScanContext().GetRegionsOfInstance() { + data.BasicRuntimeStats.tiflashScanContext.regionsOfInstance[instance.GetInstanceId()] = instance.GetRegionNum() + } data.BasicRuntimeStats.loop.Store(int32(*summary.NumIterations)) data.BasicRuntimeStats.consume.Store(int64(*summary.TimeProcessedNs)) data.BasicRuntimeStats.rows.Store(int64(*summary.NumProducedRows)) @@ -613,7 +635,9 @@ func (crs *CopRuntimeStats) GetActRows() (totalRows int64) { // MergeBasicStats traverses basicCopRuntimeStats in the CopRuntimeStats and collects some useful information. func (crs *CopRuntimeStats) MergeBasicStats() (procTimes Percentile[Duration], totalTime time.Duration, totalTasks, totalLoops, totalThreads int32, totalTiFlashScanContext TiFlashScanContext) { - totalTiFlashScanContext = TiFlashScanContext{} + totalTiFlashScanContext = TiFlashScanContext{ + regionsOfInstance: make(map[string]uint64), + } for _, instanceStats := range crs.stats { procTimes.MergePercentile(&instanceStats.procTimes) totalTime += time.Duration(instanceStats.consume.Load()) @@ -719,60 +743,227 @@ type RuntimeStats interface { // TiFlashScanContext is used to express the table scan information in tiflash type TiFlashScanContext struct { - totalDmfileScannedPacks uint64 - totalDmfileScannedRows uint64 - totalDmfileSkippedPacks uint64 - totalDmfileSkippedRows uint64 - totalDmfileRoughSetIndexCheckTimeMs uint64 - totalDmfileReadTimeMs uint64 - totalCreateSnapshotTimeMs uint64 - totalLocalRegionNum uint64 - totalRemoteRegionNum uint64 - totalLearnerReadMs uint64 - totalDisaggReadCacheHitSize uint64 - totalDisaggReadCacheMissSize uint64 + dmfileDataScannedRows uint64 + dmfileDataSkippedRows uint64 + dmfileMvccScannedRows uint64 + dmfileMvccSkippedRows uint64 + dmfileLmFilterScannedRows uint64 + dmfileLmFilterSkippedRows uint64 + totalDmfileRsCheckMs uint64 + totalDmfileReadMs uint64 + totalBuildSnapshotMs uint64 + localRegions uint64 + remoteRegions uint64 + totalLearnerReadMs uint64 + disaggReadCacheHitBytes uint64 + disaggReadCacheMissBytes uint64 + segments uint64 + readTasks uint64 + deltaRows uint64 + deltaBytes uint64 + mvccInputRows uint64 + mvccInputBytes uint64 + mvccOutputRows uint64 + lmSkipRows uint64 + totalBuildBitmapMs uint64 + totalBuildInputStreamMs uint64 + staleReadRegions uint64 + minLocalStreamMs uint64 + maxLocalStreamMs uint64 + minRemoteStreamMs uint64 + maxRemoteStreamMs uint64 + regionsOfInstance map[string]uint64 } // Clone implements the deep copy of * TiFlashshScanContext func (context *TiFlashScanContext) Clone() TiFlashScanContext { - return TiFlashScanContext{ - totalDmfileScannedPacks: context.totalDmfileScannedPacks, - totalDmfileScannedRows: context.totalDmfileScannedRows, - totalDmfileSkippedPacks: context.totalDmfileSkippedPacks, - totalDmfileSkippedRows: context.totalDmfileSkippedRows, - totalDmfileRoughSetIndexCheckTimeMs: context.totalDmfileRoughSetIndexCheckTimeMs, - totalDmfileReadTimeMs: context.totalDmfileReadTimeMs, - totalCreateSnapshotTimeMs: context.totalCreateSnapshotTimeMs, - totalLocalRegionNum: context.totalLocalRegionNum, - totalRemoteRegionNum: context.totalRemoteRegionNum, - totalLearnerReadMs: context.totalLearnerReadMs, - totalDisaggReadCacheHitSize: context.totalDisaggReadCacheHitSize, - totalDisaggReadCacheMissSize: context.totalDisaggReadCacheMissSize, - } + newContext := TiFlashScanContext{ + dmfileDataScannedRows: context.dmfileDataScannedRows, + dmfileDataSkippedRows: context.dmfileDataSkippedRows, + dmfileMvccScannedRows: context.dmfileMvccScannedRows, + dmfileMvccSkippedRows: context.dmfileMvccSkippedRows, + dmfileLmFilterScannedRows: context.dmfileLmFilterScannedRows, + dmfileLmFilterSkippedRows: context.dmfileLmFilterSkippedRows, + totalDmfileRsCheckMs: context.totalDmfileRsCheckMs, + totalDmfileReadMs: context.totalDmfileReadMs, + totalBuildSnapshotMs: context.totalBuildSnapshotMs, + localRegions: context.localRegions, + remoteRegions: context.remoteRegions, + totalLearnerReadMs: context.totalLearnerReadMs, + disaggReadCacheHitBytes: context.disaggReadCacheHitBytes, + disaggReadCacheMissBytes: context.disaggReadCacheMissBytes, + segments: context.segments, + readTasks: context.readTasks, + deltaRows: context.deltaRows, + deltaBytes: context.deltaBytes, + mvccInputRows: context.mvccInputRows, + mvccInputBytes: context.mvccInputBytes, + mvccOutputRows: context.mvccOutputRows, + lmSkipRows: context.lmSkipRows, + totalBuildBitmapMs: context.totalBuildBitmapMs, + totalBuildInputStreamMs: context.totalBuildInputStreamMs, + staleReadRegions: context.staleReadRegions, + minLocalStreamMs: context.minLocalStreamMs, + maxLocalStreamMs: context.maxLocalStreamMs, + minRemoteStreamMs: context.minRemoteStreamMs, + maxRemoteStreamMs: context.maxRemoteStreamMs, + regionsOfInstance: make(map[string]uint64), + } + for k, v := range context.regionsOfInstance { + newContext.regionsOfInstance[k] = v + } + return newContext } + func (context *TiFlashScanContext) String() string { - return fmt.Sprintf("tiflash_scan:{dtfile:{total_scanned_packs:%d, total_skipped_packs:%d, total_scanned_rows:%d, total_skipped_rows:%d, total_rs_index_check_time: %dms, total_read_time: %dms, total_disagg_read_cache_hit_size: %d, total_disagg_read_cache_miss_size: %d}, total_create_snapshot_time: %dms, total_local_region_num: %d, total_remote_region_num: %d, total_learner_read_time: %dms}", context.totalDmfileScannedPacks, context.totalDmfileSkippedPacks, context.totalDmfileScannedRows, context.totalDmfileSkippedRows, context.totalDmfileRoughSetIndexCheckTimeMs, context.totalDmfileReadTimeMs, context.totalDisaggReadCacheHitSize, context.totalDisaggReadCacheMissSize, context.totalCreateSnapshotTimeMs, context.totalLocalRegionNum, context.totalRemoteRegionNum, context.totalLearnerReadMs) + regionBalanceInfo := "none" + if len(context.regionsOfInstance) > 0 { + maxNum := uint64(0) + minNum := uint64(math.MaxUint64) + for _, v := range context.regionsOfInstance { + if v > maxNum { + maxNum = v + } + if v > 0 && v < minNum { + minNum = v + } + } + regionBalanceInfo = fmt.Sprintf("{instance_num: %d, max/min: %d/%d=%f}", + len(context.regionsOfInstance), + maxNum, + minNum, + float64(maxNum)/float64(minNum)) + } + dmfileDisaggInfo := "" + if context.disaggReadCacheHitBytes != 0 || context.disaggReadCacheMissBytes != 0 { + dmfileDisaggInfo = fmt.Sprintf(", disagg_cache_hit_bytes: %d, disagg_cache_miss_bytes: %d", + context.disaggReadCacheHitBytes, + context.disaggReadCacheMissBytes) + } + remoteStreamInfo := "" + if context.minRemoteStreamMs != 0 || context.maxRemoteStreamMs != 0 { + remoteStreamInfo = fmt.Sprintf("min_remote_stream:%dms, max_remote_stream:%dms, ", context.minRemoteStreamMs, context.maxRemoteStreamMs) + } + // note: "tot" is short for "total" + return fmt.Sprintf("tiflash_scan:{"+ + "mvcc_input_rows:%d, "+ + "mvcc_input_bytes:%d, "+ + "mvcc_output_rows:%d, "+ + "lm_skip_rows:%d, "+ + "local_regions:%d, "+ + "remote_regions:%d, "+ + "tot_learner_read:%dms, "+ + "region_balance:%s, "+ + "delta_rows:%d, "+ + "delta_bytes:%d, "+ + "segments:%d, "+ + "stale_read_regions:%d, "+ + "tot_build_snapshot:%dms, "+ + "tot_build_bitmap:%dms, "+ + "tot_build_inputstream:%dms, "+ + "min_local_stream:%dms, "+ + "max_local_stream:%dms, "+ + "%s"+ // remote stream info + "dtfile:{"+ + "data_scanned_rows:%d, "+ + "data_skipped_rows:%d, "+ + "mvcc_scanned_rows:%d, "+ + "mvcc_skipped_rows:%d, "+ + "lm_filter_scanned_rows:%d, "+ + "lm_filter_skipped_rows:%d, "+ + "tot_rs_index_check:%dms, "+ + "tot_read:%dms"+ + "%s}"+ // Disagg cache info of DMFile + "}", + context.mvccInputRows, + context.mvccInputBytes, + context.mvccOutputRows, + context.lmSkipRows, + context.localRegions, + context.remoteRegions, + context.totalLearnerReadMs, + regionBalanceInfo, + context.deltaRows, + context.deltaBytes, + context.segments, + context.staleReadRegions, + context.totalBuildSnapshotMs, + context.totalBuildBitmapMs, + context.totalBuildInputStreamMs, + context.minLocalStreamMs, + context.maxLocalStreamMs, + remoteStreamInfo, + context.dmfileDataScannedRows, + context.dmfileDataSkippedRows, + context.dmfileMvccScannedRows, + context.dmfileMvccSkippedRows, + context.dmfileLmFilterScannedRows, + context.dmfileLmFilterSkippedRows, + context.totalDmfileRsCheckMs, + context.totalDmfileReadMs, + dmfileDisaggInfo, + ) } // Merge make sum to merge the information in TiFlashScanContext func (context *TiFlashScanContext) Merge(other TiFlashScanContext) { - context.totalDmfileScannedPacks += other.totalDmfileScannedPacks - context.totalDmfileScannedRows += other.totalDmfileScannedRows - context.totalDmfileSkippedPacks += other.totalDmfileSkippedPacks - context.totalDmfileSkippedRows += other.totalDmfileSkippedRows - context.totalDmfileRoughSetIndexCheckTimeMs += other.totalDmfileRoughSetIndexCheckTimeMs - context.totalDmfileReadTimeMs += other.totalDmfileReadTimeMs - context.totalCreateSnapshotTimeMs += other.totalCreateSnapshotTimeMs - context.totalLocalRegionNum += other.totalLocalRegionNum - context.totalRemoteRegionNum += other.totalRemoteRegionNum + context.dmfileDataScannedRows += other.dmfileDataScannedRows + context.dmfileDataSkippedRows += other.dmfileDataSkippedRows + context.dmfileMvccScannedRows += other.dmfileMvccScannedRows + context.dmfileMvccSkippedRows += other.dmfileMvccSkippedRows + context.dmfileLmFilterScannedRows += other.dmfileLmFilterScannedRows + context.dmfileLmFilterSkippedRows += other.dmfileLmFilterSkippedRows + context.totalDmfileRsCheckMs += other.totalDmfileRsCheckMs + context.totalDmfileReadMs += other.totalDmfileReadMs + context.totalBuildSnapshotMs += other.totalBuildSnapshotMs + context.localRegions += other.localRegions + context.remoteRegions += other.remoteRegions context.totalLearnerReadMs += other.totalLearnerReadMs - context.totalDisaggReadCacheHitSize += other.totalDisaggReadCacheHitSize - context.totalDisaggReadCacheMissSize += other.totalDisaggReadCacheMissSize + context.disaggReadCacheHitBytes += other.disaggReadCacheHitBytes + context.disaggReadCacheMissBytes += other.disaggReadCacheMissBytes + context.segments += other.segments + context.readTasks += other.readTasks + context.deltaRows += other.deltaRows + context.deltaBytes += other.deltaBytes + context.mvccInputRows += other.mvccInputRows + context.mvccInputBytes += other.mvccInputBytes + context.mvccOutputRows += other.mvccOutputRows + context.lmSkipRows += other.lmSkipRows + context.totalBuildBitmapMs += other.totalBuildBitmapMs + context.totalBuildInputStreamMs += other.totalBuildInputStreamMs + context.staleReadRegions += other.staleReadRegions + + if context.minLocalStreamMs == 0 || other.minLocalStreamMs < context.minLocalStreamMs { + context.minLocalStreamMs = other.minLocalStreamMs + } + if other.maxLocalStreamMs > context.maxLocalStreamMs { + context.maxLocalStreamMs = other.maxLocalStreamMs + } + if context.minRemoteStreamMs == 0 || other.minRemoteStreamMs < context.minRemoteStreamMs { + context.minRemoteStreamMs = other.minRemoteStreamMs + } + if other.maxRemoteStreamMs > context.maxRemoteStreamMs { + context.maxRemoteStreamMs = other.maxRemoteStreamMs + } + + if context.regionsOfInstance == nil { + context.regionsOfInstance = make(map[string]uint64) + } + for k, v := range other.regionsOfInstance { + context.regionsOfInstance[k] += v + } } // Empty check whether TiFlashScanContext is Empty, if scan no pack and skip no pack, we regard it as empty func (context *TiFlashScanContext) Empty() bool { - res := context.totalDmfileScannedPacks == 0 && context.totalDmfileSkippedPacks == 0 && context.totalLocalRegionNum == 0 && context.totalRemoteRegionNum == 0 + res := context.dmfileDataScannedRows == 0 && + context.dmfileDataSkippedRows == 0 && + context.dmfileMvccScannedRows == 0 && + context.dmfileMvccSkippedRows == 0 && + context.dmfileLmFilterScannedRows == 0 && + context.dmfileLmFilterSkippedRows == 0 && + context.localRegions == 0 && + context.remoteRegions == 0 return res } diff --git a/pkg/util/execdetails/execdetails_test.go b/pkg/util/execdetails/execdetails_test.go index faac7100815fc..018f0fa05cfcc 100644 --- a/pkg/util/execdetails/execdetails_test.go +++ b/pkg/util/execdetails/execdetails_test.go @@ -136,20 +136,18 @@ func mockExecutorExecutionSummary(TimeProcessedNs, NumProducedRows, NumIteration NumIterations: &NumIterations, XXX_unrecognized: nil} } -func mockExecutorExecutionSummaryForTiFlash(TimeProcessedNs, NumProducedRows, NumIterations, Concurrency, totalDmfileScannedPacks, totalDmfileScannedRows, totalDmfileSkippedPacks, totalDmfileSkippedRows, totalDmfileRoughSetIndexCheckTimeMs, totalDmfileReadTimeMs, totalCreateSnapshotTimeMs, totalLocalRegionNum, totalRemoteRegionNum, totalLearnerReadMs, totalDisaggReadCacheHitSize, totalDisaggReadCacheMissSize uint64, ExecutorID string) *tipb.ExecutorExecutionSummary { +func mockExecutorExecutionSummaryForTiFlash(TimeProcessedNs, NumProducedRows, NumIterations, Concurrency, dmfileScannedRows, dmfileSkippedRows, totalDmfileRsCheckMs, totalDmfileReadTimeMs, totalBuildSnapshotMs, localRegions, remoteRegions, totalLearnerReadMs, disaggReadCacheHitBytes, disaggReadCacheMissBytes uint64, ExecutorID string) *tipb.ExecutorExecutionSummary { tiflashScanContext := tipb.TiFlashScanContext{ - TotalDmfileScannedPacks: &totalDmfileScannedPacks, - TotalDmfileSkippedPacks: &totalDmfileSkippedPacks, - TotalDmfileScannedRows: &totalDmfileScannedRows, - TotalDmfileSkippedRows: &totalDmfileSkippedRows, - TotalDmfileRoughSetIndexCheckTimeMs: &totalDmfileRoughSetIndexCheckTimeMs, - TotalDmfileReadTimeMs: &totalDmfileReadTimeMs, - TotalCreateSnapshotTimeMs: &totalCreateSnapshotTimeMs, - TotalLocalRegionNum: &totalLocalRegionNum, - TotalRemoteRegionNum: &totalRemoteRegionNum, - TotalLearnerReadMs: &totalLearnerReadMs, - TotalDisaggReadCacheHitSize: &totalDisaggReadCacheHitSize, - TotalDisaggReadCacheMissSize: &totalDisaggReadCacheMissSize, + DmfileDataScannedRows: &dmfileScannedRows, + DmfileDataSkippedRows: &dmfileSkippedRows, + TotalDmfileRsCheckMs: &totalDmfileRsCheckMs, + TotalDmfileReadMs: &totalDmfileReadTimeMs, + TotalBuildSnapshotMs: &totalBuildSnapshotMs, + LocalRegions: &localRegions, + RemoteRegions: &remoteRegions, + TotalLearnerReadMs: &totalLearnerReadMs, + DisaggReadCacheHitBytes: &disaggReadCacheHitBytes, + DisaggReadCacheMissBytes: &disaggReadCacheMissBytes, } fmt.Println("tiflashScanContext is ", tiflashScanContext.String()) return &tipb.ExecutorExecutionSummary{TimeProcessedNs: &TimeProcessedNs, NumProducedRows: &NumProducedRows, @@ -215,10 +213,10 @@ func TestCopRuntimeStatsForTiFlash(t *testing.T) { tableScanID := 1 aggID := 2 tableReaderID := 3 - stats.RecordOneCopTask(aggID, "tiflash", "8.8.8.8", mockExecutorExecutionSummaryForTiFlash(1, 1, 1, 1, 1, 8192, 0, 0, 15, 200, 40, 10, 4, 1, 100, 50, "tablescan_"+strconv.Itoa(tableScanID))) - stats.RecordOneCopTask(aggID, "tiflash", "8.8.8.9", mockExecutorExecutionSummaryForTiFlash(2, 2, 2, 1, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, "tablescan_"+strconv.Itoa(tableScanID))) - stats.RecordOneCopTask(tableScanID, "tiflash", "8.8.8.8", mockExecutorExecutionSummaryForTiFlash(3, 3, 3, 1, 2, 12000, 1, 6000, 60, 1000, 20, 5, 1, 0, 20, 0, "aggregation_"+strconv.Itoa(aggID))) - stats.RecordOneCopTask(tableScanID, "tiflash", "8.8.8.9", mockExecutorExecutionSummaryForTiFlash(4, 4, 4, 1, 1, 8192, 10, 80000, 40, 2000, 30, 1, 1, 0, 0, 0, "aggregation_"+strconv.Itoa(aggID))) + stats.RecordOneCopTask(aggID, "tiflash", "8.8.8.8", mockExecutorExecutionSummaryForTiFlash(1, 1, 1, 1, 8192, 0, 15, 200, 40, 10, 4, 1, 100, 50, "tablescan_"+strconv.Itoa(tableScanID))) + stats.RecordOneCopTask(aggID, "tiflash", "8.8.8.9", mockExecutorExecutionSummaryForTiFlash(2, 2, 2, 1, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, "tablescan_"+strconv.Itoa(tableScanID))) + stats.RecordOneCopTask(tableScanID, "tiflash", "8.8.8.8", mockExecutorExecutionSummaryForTiFlash(3, 3, 3, 1, 12000, 6000, 60, 1000, 20, 5, 1, 0, 20, 0, "aggregation_"+strconv.Itoa(aggID))) + stats.RecordOneCopTask(tableScanID, "tiflash", "8.8.8.9", mockExecutorExecutionSummaryForTiFlash(4, 4, 4, 1, 8192, 80000, 40, 2000, 30, 1, 1, 0, 0, 0, "aggregation_"+strconv.Itoa(aggID))) scanDetail := &util.ScanDetail{ TotalKeys: 10, ProcessedKeys: 10, @@ -232,15 +230,15 @@ func TestCopRuntimeStatsForTiFlash(t *testing.T) { require.True(t, stats.ExistsCopStats(tableScanID)) cop := stats.GetOrCreateCopStats(tableScanID, "tiflash") - require.Equal(t, "tiflash_task:{proc max:2ns, min:1ns, avg: 1ns, p80:2ns, p95:2ns, iters:3, tasks:2, threads:2}, tiflash_scan:{dtfile:{total_scanned_packs:1, total_skipped_packs:0, total_scanned_rows:8192, total_skipped_rows:0, total_rs_index_check_time: 15ms, total_read_time: 202ms, total_disagg_read_cache_hit_size: 100, total_disagg_read_cache_miss_size: 50}, total_create_snapshot_time: 40ms, total_local_region_num: 10, total_remote_region_num: 4, total_learner_read_time: 1ms}", cop.String()) + require.Equal(t, "tiflash_task:{proc max:2ns, min:1ns, avg: 1ns, p80:2ns, p95:2ns, iters:3, tasks:2, threads:2}, tiflash_scan:{mvcc_input_rows:0, mvcc_input_bytes:0, mvcc_output_rows:0, lm_skip_rows:0, local_regions:10, remote_regions:4, tot_learner_read:1ms, region_balance:none, delta_rows:0, delta_bytes:0, segments:0, stale_read_regions:0, tot_build_snapshot:40ms, tot_build_bitmap:0ms, tot_build_inputstream:0ms, min_local_stream:0ms, max_local_stream:0ms, dtfile:{data_scanned_rows:8192, data_skipped_rows:0, mvcc_scanned_rows:0, mvcc_skipped_rows:0, lm_filter_scanned_rows:0, lm_filter_skipped_rows:0, tot_rs_index_check:15ms, tot_read:202ms, disagg_cache_hit_bytes: 100, disagg_cache_miss_bytes: 50}}", cop.String()) copStats := cop.stats["8.8.8.8"] require.NotNil(t, copStats) copStats.SetRowNum(10) copStats.Record(time.Second, 10) - require.Equal(t, "time:1s, loops:2, threads:1, tiflash_scan:{dtfile:{total_scanned_packs:1, total_skipped_packs:0, total_scanned_rows:8192, total_skipped_rows:0, total_rs_index_check_time: 15ms, total_read_time: 200ms, total_disagg_read_cache_hit_size: 100, total_disagg_read_cache_miss_size: 50}, total_create_snapshot_time: 40ms, total_local_region_num: 10, total_remote_region_num: 4, total_learner_read_time: 1ms}", copStats.String()) - expected := "tiflash_task:{proc max:4ns, min:3ns, avg: 3ns, p80:4ns, p95:4ns, iters:7, tasks:2, threads:2}, tiflash_scan:{dtfile:{total_scanned_packs:3, total_skipped_packs:11, total_scanned_rows:20192, total_skipped_rows:86000, total_rs_index_check_time: 100ms, total_read_time: 3000ms, total_disagg_read_cache_hit_size: 20, total_disagg_read_cache_miss_size: 0}, total_create_snapshot_time: 50ms, total_local_region_num: 6, total_remote_region_num: 2, total_learner_read_time: 0ms}" + require.Equal(t, "time:1s, loops:2, threads:1, tiflash_scan:{mvcc_input_rows:0, mvcc_input_bytes:0, mvcc_output_rows:0, lm_skip_rows:0, local_regions:10, remote_regions:4, tot_learner_read:1ms, region_balance:none, delta_rows:0, delta_bytes:0, segments:0, stale_read_regions:0, tot_build_snapshot:40ms, tot_build_bitmap:0ms, tot_build_inputstream:0ms, min_local_stream:0ms, max_local_stream:0ms, dtfile:{data_scanned_rows:8192, data_skipped_rows:0, mvcc_scanned_rows:0, mvcc_skipped_rows:0, lm_filter_scanned_rows:0, lm_filter_skipped_rows:0, tot_rs_index_check:15ms, tot_read:200ms, disagg_cache_hit_bytes: 100, disagg_cache_miss_bytes: 50}}", copStats.String()) + expected := "tiflash_task:{proc max:4ns, min:3ns, avg: 3ns, p80:4ns, p95:4ns, iters:7, tasks:2, threads:2}, tiflash_scan:{mvcc_input_rows:0, mvcc_input_bytes:0, mvcc_output_rows:0, lm_skip_rows:0, local_regions:6, remote_regions:2, tot_learner_read:0ms, region_balance:none, delta_rows:0, delta_bytes:0, segments:0, stale_read_regions:0, tot_build_snapshot:50ms, tot_build_bitmap:0ms, tot_build_inputstream:0ms, min_local_stream:0ms, max_local_stream:0ms, dtfile:{data_scanned_rows:20192, data_skipped_rows:86000, mvcc_scanned_rows:0, mvcc_skipped_rows:0, lm_filter_scanned_rows:0, lm_filter_skipped_rows:0, tot_rs_index_check:100ms, tot_read:3000ms, disagg_cache_hit_bytes: 20, disagg_cache_miss_bytes: 0}}" require.Equal(t, expected, stats.GetOrCreateCopStats(aggID, "tiflash").String()) rootStats := stats.GetRootStats(tableReaderID)