fix(backup): customize the timeout of backup binary execution

Add a default setting `backupExecutionTimeout` ref: longhorn/longhorn 8954,8319 Signed-off-by: James Lu <[email protected]>
james-munson · Sep 6, 2024 · eddb7e9 · eddb7e9
1 parent 1cd29fd
commit eddb7e9
Show file tree

Hide file tree

Showing 3 changed files with 41 additions and 10 deletions.
diff --git a/controller/backup_target_controller.go b/controller/backup_target_controller.go
@@ -276,7 +276,14 @@ func newBackupTargetClient(ds *datastore.DataStore, backupTarget *longhorn.Backu
 			return nil, err
 		}
 	}
-	return engineapi.NewBackupTargetClient(engineImage, backupTarget.Spec.BackupTargetURL, credential), nil
+
+	executeTimeout, err := ds.GetSettingAsInt(types.SettingNameBackupExecutionTimeout)
+	if err != nil {
+		return nil, err
+	}
+	timeout := time.Duration(executeTimeout) * time.Minute
+
+	return engineapi.NewBackupTargetClient(engineImage, backupTarget.Spec.BackupTargetURL, credential, timeout), nil
 }
 
 func newBackupTargetClientFromDefaultEngineImage(ds *datastore.DataStore, backupTarget *longhorn.BackupTarget) (*engineapi.BackupTargetClient, error) {

diff --git a/engineapi/backups.go b/engineapi/backups.go
@@ -31,17 +31,19 @@ const (
 )
 
 type BackupTargetClient struct {
-	Image      string
-	URL        string
-	Credential map[string]string
+	Image          string
+	URL            string
+	Credential     map[string]string
+	ExecuteTimeout time.Duration
 }
 
 // NewBackupTargetClient returns the backup target client
-func NewBackupTargetClient(engineImage, url string, credential map[string]string) *BackupTargetClient {
+func NewBackupTargetClient(engineImage, url string, credential map[string]string, executeTimeout time.Duration) *BackupTargetClient {
 	return &BackupTargetClient{
-		Image:      engineImage,
-		URL:        url,
-		Credential: credential,
+		Image:          engineImage,
+		URL:            url,
+		Credential:     credential,
+		ExecuteTimeout: executeTimeout,
 	}
 }
 
@@ -68,7 +70,13 @@ func NewBackupTargetClientFromBackupTarget(backupTarget *longhorn.BackupTarget,
 		}
 	}
 
-	return NewBackupTargetClient(defaultEngineImage, backupTarget.Spec.BackupTargetURL, credential), nil
+	executeTimeout, err := ds.GetSettingAsInt(types.SettingNameBackupExecutionTimeout)
+	if err != nil {
+		return nil, err
+	}
+	timeout := time.Duration(executeTimeout) * time.Minute
+
+	return NewBackupTargetClient(defaultEngineImage, backupTarget.Spec.BackupTargetURL, credential, timeout), nil
 }
 
 func (btc *BackupTargetClient) LonghornEngineBinary() string {
@@ -130,7 +138,7 @@ func (btc *BackupTargetClient) ExecuteEngineBinary(args ...string) (string, erro
 	if err != nil {
 		return "", err
 	}
-	return lhexec.NewExecutor().Execute(envs, btc.LonghornEngineBinary(), args, lhtypes.ExecuteDefaultTimeout)
+	return lhexec.NewExecutor().Execute(envs, btc.LonghornEngineBinary(), args, btc.ExecuteTimeout)
 }
 
 func (btc *BackupTargetClient) ExecuteEngineBinaryWithTimeout(timeout time.Duration, args ...string) (string, error) {

diff --git a/types/setting.go b/types/setting.go
@@ -138,6 +138,7 @@ const (
 	SettingNameFreezeFilesystemForSnapshot                              = SettingName("freeze-filesystem-for-snapshot")
 	SettingNameAutoCleanupSnapshotWhenDeleteBackup                      = SettingName("auto-cleanup-when-delete-backup")
 	SettingNameDefaultMinNumberOfBackingImageCopies                     = SettingName("default-min-number-of-backing-image-copies")
+	SettingNameBackupExecutionTimeout                                   = SettingName("backup-execution-timeout")
 	SettingNameRWXVolumeFastFailover                                    = SettingName("rwx-volume-fast-failover")
 )
 
@@ -231,6 +232,7 @@ var (
 		SettingNameFreezeFilesystemForSnapshot,
 		SettingNameAutoCleanupSnapshotWhenDeleteBackup,
 		SettingNameDefaultMinNumberOfBackingImageCopies,
+		SettingNameBackupExecutionTimeout,
 		SettingNameRWXVolumeFastFailover,
 	}
 )
@@ -352,6 +354,7 @@ var (
 		SettingNameFreezeFilesystemForSnapshot:                              SettingDefinitionFreezeFilesystemForSnapshot,
 		SettingNameAutoCleanupSnapshotWhenDeleteBackup:                      SettingDefinitionAutoCleanupSnapshotWhenDeleteBackup,
 		SettingNameDefaultMinNumberOfBackingImageCopies:                     SettingDefinitionDefaultMinNumberOfBackingImageCopies,
+		SettingNameBackupExecutionTimeout:                                   SettingDefinitionBackupExecutionTimeout,
 		SettingNameRWXVolumeFastFailover:                                    SettingDefinitionRWXVolumeFastFailover,
 	}
 
@@ -414,6 +417,19 @@ var (
 		},
 	}
 
+	SettingDefinitionBackupExecutionTimeout = SettingDefinition{
+		DisplayName: "Backup Execution Timeout",
+		Description: "Number of minutes that Longhorn allows for the backup execution. The default value is 1.",
+		Category:    SettingCategoryBackup,
+		Type:        SettingTypeInt,
+		Required:    true,
+		ReadOnly:    false,
+		Default:     "1",
+		ValueIntRange: map[string]int{
+			ValueIntRangeMinimum: 1,
+		},
+	}
+
 	SettingDefinitionRestoreVolumeRecurringJobs = SettingDefinition{
 		DisplayName: "Restore Volume Recurring Jobs",
 		Description: "Restore recurring jobs from the backup volume on the backup target and create recurring jobs if not exist during a backup restoration.\n\n" +