From eddb7e935a41974ec3e5e4e6a7f6ec423e09ad0d Mon Sep 17 00:00:00 2001
From: James Lu <james.lu@suse.com>
Date: Fri, 6 Sep 2024 10:26:32 +0800
Subject: [PATCH] fix(backup): customize the timeout of backup binary execution

Add a default setting `backupExecutionTimeout`

ref: longhorn/longhorn 8954,8319

Signed-off-by: James Lu <james.lu@suse.com>
---
 controller/backup_target_controller.go |  9 ++++++++-
 engineapi/backups.go                   | 26 +++++++++++++++++---------
 types/setting.go                       | 16 ++++++++++++++++
 3 files changed, 41 insertions(+), 10 deletions(-)

diff --git a/controller/backup_target_controller.go b/controller/backup_target_controller.go
index b3f73af759..b1adc8c9d6 100644
--- a/controller/backup_target_controller.go
+++ b/controller/backup_target_controller.go
@@ -276,7 +276,14 @@ func newBackupTargetClient(ds *datastore.DataStore, backupTarget *longhorn.Backu
 			return nil, err
 		}
 	}
-	return engineapi.NewBackupTargetClient(engineImage, backupTarget.Spec.BackupTargetURL, credential), nil
+
+	executeTimeout, err := ds.GetSettingAsInt(types.SettingNameBackupExecutionTimeout)
+	if err != nil {
+		return nil, err
+	}
+	timeout := time.Duration(executeTimeout) * time.Minute
+
+	return engineapi.NewBackupTargetClient(engineImage, backupTarget.Spec.BackupTargetURL, credential, timeout), nil
 }
 
 func newBackupTargetClientFromDefaultEngineImage(ds *datastore.DataStore, backupTarget *longhorn.BackupTarget) (*engineapi.BackupTargetClient, error) {
diff --git a/engineapi/backups.go b/engineapi/backups.go
index aab84934c3..21ae9132df 100644
--- a/engineapi/backups.go
+++ b/engineapi/backups.go
@@ -31,17 +31,19 @@ const (
 )
 
 type BackupTargetClient struct {
-	Image      string
-	URL        string
-	Credential map[string]string
+	Image          string
+	URL            string
+	Credential     map[string]string
+	ExecuteTimeout time.Duration
 }
 
 // NewBackupTargetClient returns the backup target client
-func NewBackupTargetClient(engineImage, url string, credential map[string]string) *BackupTargetClient {
+func NewBackupTargetClient(engineImage, url string, credential map[string]string, executeTimeout time.Duration) *BackupTargetClient {
 	return &BackupTargetClient{
-		Image:      engineImage,
-		URL:        url,
-		Credential: credential,
+		Image:          engineImage,
+		URL:            url,
+		Credential:     credential,
+		ExecuteTimeout: executeTimeout,
 	}
 }
 
@@ -68,7 +70,13 @@ func NewBackupTargetClientFromBackupTarget(backupTarget *longhorn.BackupTarget,
 		}
 	}
 
-	return NewBackupTargetClient(defaultEngineImage, backupTarget.Spec.BackupTargetURL, credential), nil
+	executeTimeout, err := ds.GetSettingAsInt(types.SettingNameBackupExecutionTimeout)
+	if err != nil {
+		return nil, err
+	}
+	timeout := time.Duration(executeTimeout) * time.Minute
+
+	return NewBackupTargetClient(defaultEngineImage, backupTarget.Spec.BackupTargetURL, credential, timeout), nil
 }
 
 func (btc *BackupTargetClient) LonghornEngineBinary() string {
@@ -130,7 +138,7 @@ func (btc *BackupTargetClient) ExecuteEngineBinary(args ...string) (string, erro
 	if err != nil {
 		return "", err
 	}
-	return lhexec.NewExecutor().Execute(envs, btc.LonghornEngineBinary(), args, lhtypes.ExecuteDefaultTimeout)
+	return lhexec.NewExecutor().Execute(envs, btc.LonghornEngineBinary(), args, btc.ExecuteTimeout)
 }
 
 func (btc *BackupTargetClient) ExecuteEngineBinaryWithTimeout(timeout time.Duration, args ...string) (string, error) {
diff --git a/types/setting.go b/types/setting.go
index c7d38f61f7..01156cf281 100644
--- a/types/setting.go
+++ b/types/setting.go
@@ -138,6 +138,7 @@ const (
 	SettingNameFreezeFilesystemForSnapshot                              = SettingName("freeze-filesystem-for-snapshot")
 	SettingNameAutoCleanupSnapshotWhenDeleteBackup                      = SettingName("auto-cleanup-when-delete-backup")
 	SettingNameDefaultMinNumberOfBackingImageCopies                     = SettingName("default-min-number-of-backing-image-copies")
+	SettingNameBackupExecutionTimeout                                   = SettingName("backup-execution-timeout")
 	SettingNameRWXVolumeFastFailover                                    = SettingName("rwx-volume-fast-failover")
 )
 
@@ -231,6 +232,7 @@ var (
 		SettingNameFreezeFilesystemForSnapshot,
 		SettingNameAutoCleanupSnapshotWhenDeleteBackup,
 		SettingNameDefaultMinNumberOfBackingImageCopies,
+		SettingNameBackupExecutionTimeout,
 		SettingNameRWXVolumeFastFailover,
 	}
 )
@@ -352,6 +354,7 @@ var (
 		SettingNameFreezeFilesystemForSnapshot:                              SettingDefinitionFreezeFilesystemForSnapshot,
 		SettingNameAutoCleanupSnapshotWhenDeleteBackup:                      SettingDefinitionAutoCleanupSnapshotWhenDeleteBackup,
 		SettingNameDefaultMinNumberOfBackingImageCopies:                     SettingDefinitionDefaultMinNumberOfBackingImageCopies,
+		SettingNameBackupExecutionTimeout:                                   SettingDefinitionBackupExecutionTimeout,
 		SettingNameRWXVolumeFastFailover:                                    SettingDefinitionRWXVolumeFastFailover,
 	}
 
@@ -414,6 +417,19 @@ var (
 		},
 	}
 
+	SettingDefinitionBackupExecutionTimeout = SettingDefinition{
+		DisplayName: "Backup Execution Timeout",
+		Description: "Number of minutes that Longhorn allows for the backup execution. The default value is 1.",
+		Category:    SettingCategoryBackup,
+		Type:        SettingTypeInt,
+		Required:    true,
+		ReadOnly:    false,
+		Default:     "1",
+		ValueIntRange: map[string]int{
+			ValueIntRangeMinimum: 1,
+		},
+	}
+
 	SettingDefinitionRestoreVolumeRecurringJobs = SettingDefinition{
 		DisplayName: "Restore Volume Recurring Jobs",
 		Description: "Restore recurring jobs from the backup volume on the backup target and create recurring jobs if not exist during a backup restoration.\n\n" +