From 0e539a0b86ab42afdcc0d21c0d19830b0b8f00e9 Mon Sep 17 00:00:00 2001 From: wayblink Date: Thu, 27 Jun 2024 20:47:20 +0800 Subject: [PATCH] Support backup progress (#370) Signed-off-by: wayblink --- core/backup_context.go | 3 +-- core/backup_impl_create_backup.go | 2 +- core/backup_meta.go | 17 +++-------------- core/backup_meta_manager.go | 27 ++++++++++++++++++++++++--- 4 files changed, 29 insertions(+), 20 deletions(-) diff --git a/core/backup_context.go b/core/backup_context.go index 7866270b..5d7a82ec 100644 --- a/core/backup_context.go +++ b/core/backup_context.go @@ -279,8 +279,7 @@ func (b *BackupContext) GetBackup(ctx context.Context, request *backuppb.GetBack zap.String("backupName", request.GetBackupName()), zap.String("backupId", request.GetBackupId()), zap.String("bucketName", request.GetBucketName()), - zap.String("path", request.GetPath()), - zap.Any("resp", resp)) + zap.String("path", request.GetPath())) } return resp diff --git a/core/backup_impl_create_backup.go b/core/backup_impl_create_backup.go index 244e4679..7d2cf96c 100644 --- a/core/backup_impl_create_backup.go +++ b/core/backup_impl_create_backup.go @@ -888,7 +888,7 @@ func (b *BackupContext) copySegment(ctx context.Context, backupBinlogPath string } } } - segment.Backuped = true + b.meta.UpdateSegment(segment.GetPartitionId(), segment.GetSegmentId(), setSegmentBackuped(true)) return nil } diff --git a/core/backup_meta.go b/core/backup_meta.go index f501e3de..c9462e04 100644 --- a/core/backup_meta.go +++ b/core/backup_meta.go @@ -6,7 +6,6 @@ import ( "strings" "github.com/golang/protobuf/proto" - "github.com/zilliztech/milvus-backup/core/proto/backuppb" ) @@ -287,21 +286,11 @@ func SimpleBackupResponse(input *backuppb.BackupInfoResponse) *backuppb.BackupIn // clone and remove PartitionBackups, avoid updating here every time we add a field in CollectionBackupInfo clonedCollectionBackup := proto.Clone(coll).(*backuppb.CollectionBackupInfo) clonedCollectionBackup.PartitionBackups = nil + clonedCollectionBackup.Schema = nil collections = append(collections, clonedCollectionBackup) } - simpleBackupInfo := &backuppb.BackupInfo{ - Id: backup.GetId(), - Name: backup.GetName(), - StateCode: backup.GetStateCode(), - ErrorMessage: backup.GetErrorMessage(), - BackupTimestamp: backup.GetBackupTimestamp(), - CollectionBackups: collections, - MilvusVersion: backup.GetMilvusVersion(), - StartTime: backup.GetStartTime(), - EndTime: backup.GetEndTime(), - Progress: backup.GetProgress(), - Size: backup.GetSize(), - } + simpleBackupInfo := proto.Clone(backup).(*backuppb.BackupInfo) + simpleBackupInfo.CollectionBackups = collections return &backuppb.BackupInfoResponse{ RequestId: input.GetRequestId(), Code: input.GetCode(), diff --git a/core/backup_meta_manager.go b/core/backup_meta_manager.go index 176d9df3..ef5a2cb3 100644 --- a/core/backup_meta_manager.go +++ b/core/backup_meta_manager.go @@ -4,8 +4,10 @@ import ( "sync" "github.com/golang/protobuf/proto" + "go.uber.org/zap" "github.com/zilliztech/milvus-backup/core/proto/backuppb" + "github.com/zilliztech/milvus-backup/internal/log" ) type MetaManager struct { @@ -64,6 +66,7 @@ func (meta *MetaManager) AddBackup(backup *backuppb.BackupInfo) { meta.mu.Lock() defer meta.mu.Unlock() meta.backups[backup.Id] = backup + meta.backupNameToIdDict[backup.Name] = backup.Id } func (meta *MetaManager) AddCollection(collection *backuppb.CollectionBackupInfo) { @@ -456,24 +459,42 @@ func (meta *MetaManager) GetFullMeta(id string) *backuppb.BackupInfo { return nil } collections := meta.collections[id] + var backupedSize int64 = 0 + var totalSize int64 = 0 + cloneBackup := proto.Clone(backup).(*backuppb.BackupInfo) + collectionBackups := make([]*backuppb.CollectionBackupInfo, 0) for collectionID, collection := range collections { + collectionBackup := proto.Clone(collection).(*backuppb.CollectionBackupInfo) partitionBackups := make([]*backuppb.PartitionBackupInfo, 0) for partitionID, partition := range meta.partitions[collectionID] { segmentBackups := make([]*backuppb.SegmentBackupInfo, 0) + partitionBackup := proto.Clone(partition).(*backuppb.PartitionBackupInfo) for _, segment := range meta.segments[partitionID] { segmentBackups = append(segmentBackups, proto.Clone(segment).(*backuppb.SegmentBackupInfo)) + if segment.Backuped { + backupedSize += segment.GetSize() + } + totalSize += segment.GetSize() + partitionBackup.Size = partitionBackup.Size + segment.GetSize() } - partitionBackup := proto.Clone(partition).(*backuppb.PartitionBackupInfo) partitionBackup.SegmentBackups = segmentBackups partitionBackups = append(partitionBackups, partitionBackup) + collectionBackup.Size = collectionBackup.Size + partitionBackup.Size } - collectionBackup := proto.Clone(collection).(*backuppb.CollectionBackupInfo) collectionBackup.PartitionBackups = partitionBackups collectionBackups = append(collectionBackups, collectionBackup) + cloneBackup.Size = cloneBackup.Size + collectionBackup.Size } - cloneBackup := proto.Clone(backup).(*backuppb.BackupInfo) cloneBackup.CollectionBackups = collectionBackups + cloneBackup.Progress = 1 + if totalSize != 0 { + cloneBackup.Progress = int32(backupedSize * 100 / (totalSize)) + } else { + cloneBackup.Progress = 100 + } + log.Info("backup progress", zap.Int64("backupedSize", backupedSize), zap.Int64("totalSize", totalSize), zap.Int32("progress", cloneBackup.Progress)) + return cloneBackup }