diff --git a/.wordlist-en-custom.txt b/.wordlist-en-custom.txt index 079b392da1..d49fb2e636 100644 --- a/.wordlist-en-custom.txt +++ b/.wordlist-en-custom.txt @@ -2,7 +2,6 @@ AES API's APIs ARMv -ATTACHER AcolumnName AdditionalPodAffinity AdditionalPodAntiAffinity @@ -20,7 +19,9 @@ Azurite BDR BackupConfiguration BackupList +BackupMethod BackupPhase +BackupSnapshotStatus BackupSource BackupSpec BackupStatus @@ -55,6 +56,7 @@ Cecchi CertificatesConfiguration CertificatesStatus Certmanager +ClassName ClientCASecret ClientCertsCASecret ClientReplicationSecret @@ -337,6 +339,7 @@ ServiceAccountTemplate ServiceMonitor Silvela Slonik +SnapshotOwnerReference SnapshotType Snapshotting StatefulSets @@ -366,12 +369,14 @@ Valerio VirtualBox VolumeSnapshot VolumeSnapshotClass +VolumeSnapshotConfiguration WAL WAL's WALBackupConfiguration WALs Wadle WalBackupConfiguration +WalClassName YXBw YY YYYY @@ -986,7 +991,6 @@ volumeMode volumeMounts volumeSnapshot volumesnapshot -volumesnapshotclass wal walSegmentSize walStorage diff --git a/api/v1/backup_types.go b/api/v1/backup_types.go index 16611cd52c..d04a29c822 100644 --- a/api/v1/backup_types.go +++ b/api/v1/backup_types.go @@ -17,6 +17,8 @@ limitations under the License. package v1 import ( + volumesnapshot "github.com/kubernetes-csi/external-snapshotter/client/v6/apis/volumesnapshot/v1" + corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) @@ -43,6 +45,20 @@ const ( BackupPhaseWalArchivingFailing = "walArchivingFailing" ) +// BackupMethod defines the way of executing the physical base backups of +// the selected PostgreSQL instance +type BackupMethod string + +const ( + // BackupMethodVolumeSnapshot means using the volume snapshot + // Kubernetes feature + BackupMethodVolumeSnapshot BackupMethod = "volumeSnapshot" + + // BackupMethodBarmanObjectStore means using barman to backup the + // PostgreSQL cluster + BackupMethodBarmanObjectStore BackupMethod = "barmanObjectStore" +) + // BackupSpec defines the desired state of Backup type BackupSpec struct { // The cluster to backup @@ -56,6 +72,18 @@ type BackupSpec struct { // standby, if available. // +kubebuilder:validation:Enum=primary;prefer-standby Target BackupTarget `json:"target,omitempty"` + + // The backup method to be used, possible options are `barmanObjectStore` + // and `volumeSnapshot`. Defaults to: `barmanObjectStore`. + // +kubebuilder:validation:Enum=barmanObjectStore;volumeSnapshot + // +kubebuilder:default:=barmanObjectStore + Method BackupMethod `json:"method,omitempty"` +} + +// BackupSnapshotStatus the fields exclusive to the volumeSnapshot method backup +type BackupSnapshotStatus struct { + // The snapshot lists, populated if it is a snapshot type backup + Snapshots []string `json:"snapshots,omitempty"` } // BackupStatus defines the observed state of Backup @@ -122,6 +150,12 @@ type BackupStatus struct { // Information to identify the instance where the backup has been taken from InstanceID *InstanceID `json:"instanceID,omitempty"` + + // BackupSnapshotStatus the status of to the volumeSnapshot backup + BackupSnapshotStatus BackupSnapshotStatus `json:"snapshotBackupStatus,omitempty"` + + // The backup method being used + Method BackupMethod `json:"method,omitempty"` } // InstanceID contains the information to identify an instance @@ -185,6 +219,25 @@ func (backupStatus *BackupStatus) SetAsCompleted() { backupStatus.Error = "" } +// SetAsStarted marks a certain backup as started +func (backupStatus *BackupStatus) SetAsStarted(targetPod *corev1.Pod, method BackupMethod) { + backupStatus.Phase = BackupPhaseStarted + backupStatus.InstanceID = &InstanceID{ + PodName: targetPod.Name, + ContainerID: targetPod.Status.ContainerStatuses[0].ContainerID, + } + backupStatus.Method = method +} + +// SetSnapshotList sets the Snapshots field from a list of VolumeSnapshot +func (snapshotStatus *BackupSnapshotStatus) SetSnapshotList(snapshots []*volumesnapshot.VolumeSnapshot) { + snapshotNames := make([]string, len(snapshots)) + for idx, volumeSnapshot := range snapshots { + snapshotNames[idx] = volumeSnapshot.Name + } + snapshotStatus.Snapshots = snapshotNames +} + // IsDone check if a backup is completed or still in progress func (backupStatus *BackupStatus) IsDone() bool { return backupStatus.Phase == BackupPhaseCompleted || backupStatus.Phase == BackupPhaseFailed diff --git a/api/v1/backup_types_test.go b/api/v1/backup_types_test.go new file mode 100644 index 0000000000..8830323da8 --- /dev/null +++ b/api/v1/backup_types_test.go @@ -0,0 +1,72 @@ +/* +Copyright The CloudNativePG Contributors + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package v1 + +import ( + volumesnapshot "github.com/kubernetes-csi/external-snapshotter/client/v6/apis/volumesnapshot/v1" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +var _ = Describe("BackupStatus structure", func() { + It("can be set as started", func() { + status := BackupStatus{} + pod := corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: "cluster-example-1", + }, + Status: corev1.PodStatus{ + ContainerStatuses: []corev1.ContainerStatus{ + { + ContainerID: "container-id", + }, + }, + }, + } + + status.SetAsStarted(&pod, BackupMethodBarmanObjectStore) + Expect(status.Phase).To(BeEquivalentTo(BackupPhaseStarted)) + Expect(status.InstanceID).ToNot(BeNil()) + Expect(status.InstanceID.PodName).To(Equal("cluster-example-1")) + Expect(status.InstanceID.ContainerID).To(Equal("container-id")) + Expect(status.IsDone()).To(BeFalse()) + }) + + It("can be set to contain a snapshot list", func() { + status := BackupStatus{} + status.BackupSnapshotStatus.SetSnapshotList([]*volumesnapshot.VolumeSnapshot{ + { + ObjectMeta: metav1.ObjectMeta{ + Name: "cluster-example-snapshot-1", + }, + }, + { + ObjectMeta: metav1.ObjectMeta{ + Name: "cluster-example-snapshot-2", + }, + }, + }) + + Expect(status.BackupSnapshotStatus.Snapshots).To(HaveLen(2)) + Expect(status.BackupSnapshotStatus.Snapshots).To(ConsistOf( + "cluster-example-snapshot-1", + "cluster-example-snapshot-2")) + }) +}) diff --git a/api/v1/cluster_types.go b/api/v1/cluster_types.go index 6bdf6a0f8e..4930e22169 100644 --- a/api/v1/cluster_types.go +++ b/api/v1/cluster_types.go @@ -114,6 +114,37 @@ const ( PGBouncerPoolerUserName = "cnpg_pooler_pgbouncer" ) +// SnapshotOwnerReference defines the reference type for the owner of the snapshot. +// This specifies which owner the processed resources should relate to. +type SnapshotOwnerReference string + +// Constants to represent the allowed types for SnapshotOwnerReference. +const ( + // ShapshotOwnerReferenceNone indicates that the snapshot does not have any owner reference. + ShapshotOwnerReferenceNone SnapshotOwnerReference = "none" + // SnapshotOwnerReferenceBackup indicates that the snapshot is owned by the backup resource. + SnapshotOwnerReferenceBackup SnapshotOwnerReference = "backup" + // SnapshotOwnerReferenceCluster indicates that the snapshot is owned by the cluster resource. + SnapshotOwnerReferenceCluster SnapshotOwnerReference = "cluster" +) + +// VolumeSnapshotConfiguration represents the configuration for the execution of snapshot backups. +type VolumeSnapshotConfiguration struct { + // Labels are key-value pairs that will be added to .metadata.labels snapshot resources. + Labels map[string]string `json:"labels,omitempty"` + // Annotations key-value pairs that will be added to .metadata.annotations snapshot resources. + Annotations map[string]string `json:"annotations,omitempty"` + // ClassName specifies the Snapshot Class to be used for PG_DATA PersistentVolumeClaim. + // It is the default class for the other types if no specific class is present + ClassName string `json:"className,omitempty"` + // WalClassName specifies the Snapshot Class to be used for the PG_WAL PersistentVolumeClaim. + WalClassName string `json:"walClassName,omitempty"` + // SnapshotOwnerReference indicates the type of owner reference the snapshot should have. . + // +kubebuilder:validation:Enum=none;cluster;backup + // +kubebuilder:default:=none + SnapshotOwnerReference SnapshotOwnerReference `json:"snapshotOwnerReference,omitempty"` +} + // ClusterSpec defines the desired state of Cluster type ClusterSpec struct { // Description of this PostgreSQL cluster @@ -594,6 +625,38 @@ const ( ConditionClusterReady ClusterConditionType = "Ready" ) +// A Condition that can be used to communicate the Backup progress +var ( + // BackupSucceededCondition is added to a backup + // when it was completed correctly + BackupSucceededCondition = &metav1.Condition{ + Type: string(ConditionBackup), + Status: metav1.ConditionTrue, + Reason: string(ConditionReasonLastBackupSucceeded), + Message: "Backup was successful", + } + + // BackupStartingCondition is added to a backup + // when it started + BackupStartingCondition = &metav1.Condition{ + Type: string(ConditionBackup), + Status: metav1.ConditionFalse, + Reason: string(ConditionBackupStarted), + Message: "New Backup starting up", + } + + // BuildClusterBackupFailedCondition builds + // ConditionReasonLastBackupFailed condition + BuildClusterBackupFailedCondition = func(err error) *metav1.Condition { + return &metav1.Condition{ + Type: string(ConditionBackup), + Status: metav1.ConditionFalse, + Reason: string(ConditionReasonLastBackupFailed), + Message: err.Error(), + } + } +) + // ConditionStatus defines conditions of resources type ConditionStatus string @@ -1468,10 +1531,13 @@ type BarmanObjectStoreConfiguration struct { } // BackupConfiguration defines how the backup of the cluster are taken. -// Currently the only supported backup method is barmanObjectStore. +// The supported backup methods are BarmanObjectStore and VolumeSnapshot. // For details and examples refer to the Backup and Recovery section of the // documentation type BackupConfiguration struct { + // VolumeSnapshot provides the configuration for the execution of volume snapshot backups. + VolumeSnapshot *VolumeSnapshotConfiguration `json:"volumeSnapshot,omitempty"` + // The configuration for the barman-cloud tool suite BarmanObjectStore *BarmanObjectStoreConfiguration `json:"barmanObjectStore,omitempty"` @@ -1479,6 +1545,7 @@ type BackupConfiguration struct { // and WALs (i.e. '60d'). The retention policy is expressed in the form // of `XXu` where `XX` is a positive integer and `u` is in `[dwm]` - // days, weeks, months. + // It's currently only applicable when using the BarmanObjectStore method. // +kubebuilder:validation:Pattern=^[1-9][0-9]*[dwm]$ // +optional RetentionPolicy string `json:"retentionPolicy,omitempty"` diff --git a/api/v1/scheduledbackup_types.go b/api/v1/scheduledbackup_types.go index f02ef5aa6a..526a5b8118 100644 --- a/api/v1/scheduledbackup_types.go +++ b/api/v1/scheduledbackup_types.go @@ -55,6 +55,12 @@ type ScheduledBackupSpec struct { // standby, if available. // +kubebuilder:validation:Enum=primary;prefer-standby Target BackupTarget `json:"target,omitempty"` + + // The backup method to be used, possible options are `barmanObjectStore` + // and `volumeSnapshot`. Defaults to: `barmanObjectStore`. + // +kubebuilder:validation:Enum=barmanObjectStore;volumeSnapshot + // +kubebuilder:default:=barmanObjectStore + Method BackupMethod `json:"method,omitempty"` } // ScheduledBackupStatus defines the observed state of ScheduledBackup @@ -152,6 +158,7 @@ func (scheduledBackup *ScheduledBackup) CreateBackup(name string) *Backup { Spec: BackupSpec{ Cluster: scheduledBackup.Spec.Cluster, Target: scheduledBackup.Spec.Target, + Method: scheduledBackup.Spec.Method, }, } utils.InheritAnnotations(&backup.ObjectMeta, scheduledBackup.Annotations, nil, configuration.Current) diff --git a/api/v1/zz_generated.deepcopy.go b/api/v1/zz_generated.deepcopy.go index 17be8c37b1..02787f8c78 100644 --- a/api/v1/zz_generated.deepcopy.go +++ b/api/v1/zz_generated.deepcopy.go @@ -143,6 +143,11 @@ func (in *Backup) DeepCopyObject() runtime.Object { // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *BackupConfiguration) DeepCopyInto(out *BackupConfiguration) { *out = *in + if in.VolumeSnapshot != nil { + in, out := &in.VolumeSnapshot, &out.VolumeSnapshot + *out = new(VolumeSnapshotConfiguration) + (*in).DeepCopyInto(*out) + } if in.BarmanObjectStore != nil { in, out := &in.BarmanObjectStore, &out.BarmanObjectStore *out = new(BarmanObjectStoreConfiguration) @@ -192,6 +197,26 @@ func (in *BackupList) DeepCopyObject() runtime.Object { return nil } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *BackupSnapshotStatus) DeepCopyInto(out *BackupSnapshotStatus) { + *out = *in + if in.Snapshots != nil { + in, out := &in.Snapshots, &out.Snapshots + *out = make([]string, len(*in)) + copy(*out, *in) + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new BackupSnapshotStatus. +func (in *BackupSnapshotStatus) DeepCopy() *BackupSnapshotStatus { + if in == nil { + return nil + } + out := new(BackupSnapshotStatus) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *BackupSource) DeepCopyInto(out *BackupSource) { *out = *in @@ -251,6 +276,7 @@ func (in *BackupStatus) DeepCopyInto(out *BackupStatus) { *out = new(InstanceID) **out = **in } + in.BackupSnapshotStatus.DeepCopyInto(&out.BackupSnapshotStatus) } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new BackupStatus. @@ -2063,6 +2089,35 @@ func (in *Topology) DeepCopy() *Topology { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *VolumeSnapshotConfiguration) DeepCopyInto(out *VolumeSnapshotConfiguration) { + *out = *in + if in.Labels != nil { + in, out := &in.Labels, &out.Labels + *out = make(map[string]string, len(*in)) + for key, val := range *in { + (*out)[key] = val + } + } + if in.Annotations != nil { + in, out := &in.Annotations, &out.Annotations + *out = make(map[string]string, len(*in)) + for key, val := range *in { + (*out)[key] = val + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new VolumeSnapshotConfiguration. +func (in *VolumeSnapshotConfiguration) DeepCopy() *VolumeSnapshotConfiguration { + if in == nil { + return nil + } + out := new(VolumeSnapshotConfiguration) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *WalBackupConfiguration) DeepCopyInto(out *WalBackupConfiguration) { *out = *in diff --git a/config/crd/bases/postgresql.cnpg.io_backups.yaml b/config/crd/bases/postgresql.cnpg.io_backups.yaml index a691f61a07..a40e206539 100644 --- a/config/crd/bases/postgresql.cnpg.io_backups.yaml +++ b/config/crd/bases/postgresql.cnpg.io_backups.yaml @@ -57,6 +57,14 @@ spec: required: - name type: object + method: + default: barmanObjectStore + description: 'The backup method to be used, possible options are `barmanObjectStore` + and `volumeSnapshot`. Defaults to: `barmanObjectStore`.' + enum: + - barmanObjectStore + - volumeSnapshot + type: string target: description: The policy to decide which instance should perform this backup. If empty, it defaults to `cluster.spec.backup.target`. Available @@ -222,6 +230,9 @@ spec: description: The pod name type: string type: object + method: + description: The backup method being used + type: string phase: description: The last backup status type: string @@ -290,6 +301,17 @@ spec: description: The server name on S3, the cluster name is used if this parameter is omitted type: string + snapshotBackupStatus: + description: BackupSnapshotStatus the status of to the volumeSnapshot + backup + properties: + snapshots: + description: The snapshot lists, populated if it is a snapshot + type backup + items: + type: string + type: array + type: object startedAt: description: When the backup was started format: date-time diff --git a/config/crd/bases/postgresql.cnpg.io_clusters.yaml b/config/crd/bases/postgresql.cnpg.io_clusters.yaml index d173a7671e..356f9da8f0 100644 --- a/config/crd/bases/postgresql.cnpg.io_clusters.yaml +++ b/config/crd/bases/postgresql.cnpg.io_clusters.yaml @@ -1227,7 +1227,8 @@ spec: description: RetentionPolicy is the retention policy to be used for backups and WALs (i.e. '60d'). The retention policy is expressed in the form of `XXu` where `XX` is a positive integer and `u` - is in `[dwm]` - days, weeks, months. + is in `[dwm]` - days, weeks, months. It's currently only applicable + when using the BarmanObjectStore method. pattern: ^[1-9][0-9]*[dwm]$ type: string target: @@ -1241,6 +1242,41 @@ spec: - primary - prefer-standby type: string + volumeSnapshot: + description: VolumeSnapshot provides the configuration for the + execution of volume snapshot backups. + properties: + annotations: + additionalProperties: + type: string + description: Annotations key-value pairs that will be added + to .metadata.annotations snapshot resources. + type: object + className: + description: ClassName specifies the Snapshot Class to be + used for PG_DATA PersistentVolumeClaim. It is the default + class for the other types if no specific class is present + type: string + labels: + additionalProperties: + type: string + description: Labels are key-value pairs that will be added + to .metadata.labels snapshot resources. + type: object + snapshotOwnerReference: + default: none + description: SnapshotOwnerReference indicates the type of + owner reference the snapshot should have. . + enum: + - none + - cluster + - backup + type: string + walClassName: + description: WalClassName specifies the Snapshot Class to + be used for the PG_WAL PersistentVolumeClaim. + type: string + type: object type: object bootstrap: description: Instructions to bootstrap this cluster diff --git a/config/crd/bases/postgresql.cnpg.io_scheduledbackups.yaml b/config/crd/bases/postgresql.cnpg.io_scheduledbackups.yaml index b4567c00fa..f31ac5de98 100644 --- a/config/crd/bases/postgresql.cnpg.io_scheduledbackups.yaml +++ b/config/crd/bases/postgresql.cnpg.io_scheduledbackups.yaml @@ -70,6 +70,14 @@ spec: description: If the first backup has to be immediately start after creation or not type: boolean + method: + default: barmanObjectStore + description: 'The backup method to be used, possible options are `barmanObjectStore` + and `volumeSnapshot`. Defaults to: `barmanObjectStore`.' + enum: + - barmanObjectStore + - volumeSnapshot + type: string schedule: description: The schedule does not follow the same format used in Kubernetes CronJobs as it includes an additional seconds specifier, diff --git a/config/rbac/role.yaml b/config/rbac/role.yaml index 9d81a7d592..c45326a544 100644 --- a/config/rbac/role.yaml +++ b/config/rbac/role.yaml @@ -325,3 +325,12 @@ rules: - patch - update - watch +- apiGroups: + - snapshot.storage.k8s.io + resources: + - volumesnapshots + verbs: + - create + - get + - list + - watch diff --git a/controllers/backup_controller.go b/controllers/backup_controller.go index 2a1d20116f..06d578f818 100644 --- a/controllers/backup_controller.go +++ b/controllers/backup_controller.go @@ -22,9 +22,9 @@ import ( "fmt" "time" + storagesnapshotv1 "github.com/kubernetes-csi/external-snapshotter/client/v6/apis/volumesnapshot/v1" corev1 "k8s.io/api/core/v1" apierrs "k8s.io/apimachinery/pkg/api/errors" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/types" "k8s.io/client-go/kubernetes" @@ -33,6 +33,7 @@ import ( ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/builder" "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/controller" "sigs.k8s.io/controller-runtime/pkg/event" "sigs.k8s.io/controller-runtime/pkg/handler" "sigs.k8s.io/controller-runtime/pkg/manager" @@ -43,8 +44,10 @@ import ( "github.com/cloudnative-pg/cloudnative-pg/pkg/conditions" "github.com/cloudnative-pg/cloudnative-pg/pkg/management/log" "github.com/cloudnative-pg/cloudnative-pg/pkg/management/postgres" + "github.com/cloudnative-pg/cloudnative-pg/pkg/reconciler/persistentvolumeclaim" "github.com/cloudnative-pg/cloudnative-pg/pkg/specs" "github.com/cloudnative-pg/cloudnative-pg/pkg/utils" + "github.com/cloudnative-pg/cloudnative-pg/pkg/utils/snapshot" ) // backupPhase indicates the path inside the Backup kind @@ -73,11 +76,13 @@ func NewBackupReconciler(mgr manager.Manager) *BackupReconciler { // +kubebuilder:rbac:groups=postgresql.cnpg.io,resources=backups,verbs=get;list;watch;create;update;patch;delete // +kubebuilder:rbac:groups=postgresql.cnpg.io,resources=backups/status,verbs=get;update;patch // +kubebuilder:rbac:groups=postgresql.cnpg.io,resources=clusters,verbs=get +// +kubebuilder:rbac:groups=snapshot.storage.k8s.io,resources=volumesnapshots,verbs=get;create;watch;list // +kubebuilder:rbac:groups="",resources=events,verbs=create;patch // +kubebuilder:rbac:groups="",resources=pods/exec,verbs=get;list;delete;patch;create;watch // +kubebuilder:rbac:groups="",resources=pods,verbs=get // Reconcile is the main reconciliation loop +// nolint: gocognit func (r *BackupReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { contextLogger, ctx := log.SetupLogger(ctx) contextLogger.Debug(fmt.Sprintf("reconciling object %#q", req.NamespacedName)) @@ -192,17 +197,102 @@ func (r *BackupReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctr "cluster", cluster.Name, "pod", pod.Name) - // This backup has been started - if err := StartBackup(ctx, r.Client, &backup, pod, &cluster); err != nil { - r.Recorder.Eventf(&backup, "Warning", "Error", "Backup exit with error %v", err) - tryFlagBackupAsFailed(ctx, r.Client, &backup, fmt.Errorf("encountered an error while taking the backup: %w", err)) - return ctrl.Result{}, nil + switch backup.Spec.Method { + case apiv1.BackupMethodBarmanObjectStore: + if cluster.Spec.Backup.BarmanObjectStore == nil { + tryFlagBackupAsFailed(ctx, r.Client, &backup, + errors.New("no barmanObjectStore section defined on the target cluster")) + return ctrl.Result{}, nil + } + // This backup has been started + if err := startBarmanBackup(ctx, r.Client, &backup, pod, &cluster); err != nil { + r.Recorder.Eventf(&backup, "Warning", "Error", "Backup exit with error %v", err) + tryFlagBackupAsFailed(ctx, r.Client, &backup, fmt.Errorf("encountered an error while taking the backup: %w", err)) + return ctrl.Result{}, nil + } + case apiv1.BackupMethodVolumeSnapshot: + if cluster.Spec.Backup.VolumeSnapshot == nil { + tryFlagBackupAsFailed(ctx, r.Client, &backup, + errors.New("no volumeSnapshot section defined on the target cluster")) + return ctrl.Result{}, nil + } + if err := startSnapshotBackup(ctx, r.Client, pod, &cluster, &backup); err != nil { + r.Recorder.Eventf(&backup, "Warning", "Error", "snapshot backup failed: %v", err) + tryFlagBackupAsFailed(ctx, r.Client, &backup, + fmt.Errorf("encountered an error while taking the snapshot backup: %w", err)) + return ctrl.Result{}, nil + } + default: + return ctrl.Result{}, fmt.Errorf("unrecognized method: %s", backup.Spec.Method) } contextLogger.Debug(fmt.Sprintf("object %#q has been reconciled", req.NamespacedName)) return ctrl.Result{}, nil } +func startSnapshotBackup( + ctx context.Context, + cli client.Client, + targetPod *corev1.Pod, + cluster *apiv1.Cluster, + backup *apiv1.Backup, +) error { + contextLogger := log.FromContext(ctx) + + backup.Status.SetAsStarted(targetPod, apiv1.BackupMethodVolumeSnapshot) + if err := postgres.PatchBackupStatusAndRetry(ctx, cli, backup); err != nil { + return err + } + + if errCond := conditions.Patch(ctx, cli, cluster, apiv1.BackupStartingCondition); errCond != nil { + log.FromContext(ctx).Error(errCond, "Error while updating backup condition (backup starting)") + } + + pvcs, err := persistentvolumeclaim.GetInstancePVCs(ctx, cli, targetPod.Name, cluster.Namespace) + if err != nil { + return fmt.Errorf("cannot get PVCs: %w", err) + } + + snapshotConfig := *cluster.Spec.Backup.VolumeSnapshot + + snapshotEnrich := func(vs *storagesnapshotv1.VolumeSnapshot) { + switch snapshotConfig.SnapshotOwnerReference { + case apiv1.SnapshotOwnerReferenceCluster: + cluster.SetInheritedDataAndOwnership(&vs.ObjectMeta) + case apiv1.SnapshotOwnerReferenceBackup: + utils.SetAsOwnedBy(&vs.ObjectMeta, backup.ObjectMeta, backup.TypeMeta) + default: + break + } + } + executor := snapshot. + NewExecutorBuilder(cli, snapshotConfig). + FenceInstance(true). + WithSnapshotEnrich(snapshotEnrich). + Build() + + snapshots, err := executor.Execute(ctx, cluster, targetPod, pvcs) + if err != nil { + contextLogger.Error(err, "while executing snapshot backup") + backup.Status.SetAsFailed(fmt.Errorf("can't execute snapshot backup: %w", err)) + + // Update backup status in cluster conditions + if errCond := conditions.Patch(ctx, cli, cluster, apiv1.BuildClusterBackupFailedCondition(err)); errCond != nil { + log.FromContext(ctx).Error(errCond, "Error while updating backup condition (backup snapshot failed)") + } + return postgres.PatchBackupStatusAndRetry(ctx, cli, backup) + } + + if err := conditions.Patch(ctx, cli, cluster, apiv1.BackupSucceededCondition); err != nil { + contextLogger.Error(err, "Can't update the cluster with the completed snapshot backup data") + } + + backup.Status.SetAsCompleted() + backup.Status.BackupSnapshotStatus.SetSnapshotList(snapshots) + + return postgres.PatchBackupStatusAndRetry(ctx, cli, backup) +} + // getBackupTargetPod returns the correct pod that should run the backup according to the current // cluster's target policy func (r *BackupReconciler) getBackupTargetPod(ctx context.Context, @@ -214,7 +304,10 @@ func (r *BackupReconciler) getBackupTargetPod(ctx context.Context, if err != nil { return nil, err } - backupTarget := cluster.Spec.Backup.Target + var backupTarget apiv1.BackupTarget + if cluster.Spec.Backup != nil { + backupTarget = cluster.Spec.Backup.Target + } if backup.Spec.Target != "" { backupTarget = backup.Spec.Target } @@ -252,9 +345,9 @@ func (r *BackupReconciler) getBackupTargetPod(ctx context.Context, return &pod, err } -// StartBackup request a backup in a Pod and marks the backup started +// startBarmanBackup request a backup in a Pod and marks the backup started // or failed if needed -func StartBackup( +func startBarmanBackup( ctx context.Context, client client.Client, backup *apiv1.Backup, @@ -263,8 +356,8 @@ func StartBackup( ) error { // This backup has been started status := backup.GetStatus() - status.Phase = apiv1.BackupPhaseStarted - status.InstanceID = &apiv1.InstanceID{PodName: pod.Name, ContainerID: pod.Status.ContainerStatuses[0].ContainerID} + status.SetAsStarted(pod, apiv1.BackupMethodBarmanObjectStore) + if err := postgres.PatchBackupStatusAndRetry(ctx, client, backup); err != nil { return err } @@ -295,13 +388,7 @@ func StartBackup( status.CommandError = stdout // Update backup status in cluster conditions - condition := metav1.Condition{ - Type: string(apiv1.ConditionBackup), - Status: metav1.ConditionFalse, - Reason: string(apiv1.ConditionReasonLastBackupFailed), - Message: err.Error(), - } - if errCond := conditions.Patch(ctx, client, cluster, &condition); errCond != nil { + if errCond := conditions.Patch(ctx, client, cluster, apiv1.BuildClusterBackupFailedCondition(err)); errCond != nil { log.FromContext(ctx).Error(errCond, "Error while updating backup condition (backup failed)") } return postgres.PatchBackupStatusAndRetry(ctx, client, backup) @@ -326,6 +413,7 @@ func (r *BackupReconciler) SetupWithManager(ctx context.Context, mgr ctrl.Manage handler.EnqueueRequestsFromMapFunc(r.mapClustersToBackup()), builder.WithPredicates(clustersWithBackupPredicate), ). + WithOptions(controller.Options{MaxConcurrentReconciles: 5}). Complete(r) } diff --git a/docs/src/api_reference.md b/docs/src/api_reference.md index b0841ef753..5dd0706e26 100644 --- a/docs/src/api_reference.md +++ b/docs/src/api_reference.md @@ -23,6 +23,7 @@ Below you will find a description of the defined resources: - [Backup](#Backup) - [BackupConfiguration](#BackupConfiguration) - [BackupList](#BackupList) +- [BackupSnapshotStatus](#BackupSnapshotStatus) - [BackupSource](#BackupSource) - [BackupSpec](#BackupSpec) - [BackupStatus](#BackupStatus) @@ -90,6 +91,7 @@ Below you will find a description of the defined resources: - [StorageConfiguration](#StorageConfiguration) - [SyncReplicaElectionConstraints](#SyncReplicaElectionConstraints) - [Topology](#Topology) +- [VolumeSnapshotConfiguration](#VolumeSnapshotConfiguration) - [WalBackupConfiguration](#WalBackupConfiguration) @@ -144,12 +146,13 @@ Name | Description ## BackupConfiguration -BackupConfiguration defines how the backup of the cluster are taken. Currently the only supported backup method is barmanObjectStore. For details and examples refer to the Backup and Recovery section of the documentation +BackupConfiguration defines how the backup of the cluster are taken. The supported backup methods are BarmanObjectStore and VolumeSnapshot. For details and examples refer to the Backup and Recovery section of the documentation Name | Description | Type ----------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ------------------------------------------------------------------ +`volumeSnapshot ` | VolumeSnapshot provides the configuration for the execution of volume snapshot backups. | [*VolumeSnapshotConfiguration](#VolumeSnapshotConfiguration) `barmanObjectStore` | The configuration for the barman-cloud tool suite | [*BarmanObjectStoreConfiguration](#BarmanObjectStoreConfiguration) -`retentionPolicy ` | RetentionPolicy is the retention policy to be used for backups and WALs (i.e. '60d'). The retention policy is expressed in the form of `XXu` where `XX` is a positive integer and `u` is in `[dwm]` - days, weeks, months. | string +`retentionPolicy ` | RetentionPolicy is the retention policy to be used for backups and WALs (i.e. '60d'). The retention policy is expressed in the form of `XXu` where `XX` is a positive integer and `u` is in `[dwm]` - days, weeks, months. It's currently only applicable when using the BarmanObjectStore method. | string `target ` | The policy to decide which instance should perform backups. Available options are empty string, which will default to `prefer-standby` policy, `primary` to have backups run always on primary instances, `prefer-standby` to have backups run preferably on the most updated standby, if available. | BackupTarget @@ -163,6 +166,16 @@ Name | Description `metadata` | Standard list metadata. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds | [metav1.ListMeta](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.27/#listmeta-v1-meta) `items ` | List of backups - *mandatory* | [[]Backup](#Backup) + + +## BackupSnapshotStatus + +BackupSnapshotStatus the fields exclusive to the volumeSnapshot method backup + +Name | Description | Type +--------- | ------------------------------------------------------------- | -------- +`snapshots` | The snapshot lists, populated if it is a snapshot type backup | []string + ## BackupSource @@ -183,6 +196,7 @@ Name | Description ------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | --------------------------------------------- `cluster` | The cluster to backup | [LocalObjectReference](#LocalObjectReference) `target ` | The policy to decide which instance should perform this backup. If empty, it defaults to `cluster.spec.backup.target`. Available options are empty string, `primary` and `prefer-standby`. `primary` to have backups run always on primary instances, `prefer-standby` to have backups run preferably on the most updated standby, if available. | BackupTarget +`method ` | The backup method to be used, possible options are `barmanObjectStore` and `volumeSnapshot`. Defaults to: `barmanObjectStore`. | BackupMethod @@ -190,26 +204,28 @@ Name | Description BackupStatus defines the observed state of Backup -Name | Description | Type ---------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------- -`endpointCA ` | EndpointCA store the CA bundle of the barman endpoint. Useful when using self-signed certificates to avoid errors with certificate issuer and barman-cloud-wal-archive. | [*SecretKeySelector](#SecretKeySelector) -`endpointURL ` | Endpoint to be used to upload data to the cloud, overriding the automatic endpoint discovery | string -`destinationPath` | The path where to store the backup (i.e. s3://bucket/path/to/folder) this path, with different destination folders, will be used for WALs and for data. This may not be populated in case of errors. | string -`serverName ` | The server name on S3, the cluster name is used if this parameter is omitted | string -`encryption ` | Encryption method required to S3 API | string -`backupId ` | The ID of the Barman backup | string -`backupName ` | The Name of the Barman backup | string -`phase ` | The last backup status | BackupPhase -`startedAt ` | When the backup was started | [*metav1.Time](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.27/#time-v1-meta) -`stoppedAt ` | When the backup was terminated | [*metav1.Time](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.27/#time-v1-meta) -`beginWal ` | The starting WAL | string -`endWal ` | The ending WAL | string -`beginLSN ` | The starting xlog | string -`endLSN ` | The ending xlog | string -`error ` | The detected error | string -`commandOutput ` | Unused. Retained for compatibility with old versions. | string -`commandError ` | The backup command output in case of error | string -`instanceID ` | Information to identify the instance where the backup has been taken from | [*InstanceID](#InstanceID) +Name | Description | Type +-------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------- +`endpointCA ` | EndpointCA store the CA bundle of the barman endpoint. Useful when using self-signed certificates to avoid errors with certificate issuer and barman-cloud-wal-archive. | [*SecretKeySelector](#SecretKeySelector) +`endpointURL ` | Endpoint to be used to upload data to the cloud, overriding the automatic endpoint discovery | string +`destinationPath ` | The path where to store the backup (i.e. s3://bucket/path/to/folder) this path, with different destination folders, will be used for WALs and for data. This may not be populated in case of errors. | string +`serverName ` | The server name on S3, the cluster name is used if this parameter is omitted | string +`encryption ` | Encryption method required to S3 API | string +`backupId ` | The ID of the Barman backup | string +`backupName ` | The Name of the Barman backup | string +`phase ` | The last backup status | BackupPhase +`startedAt ` | When the backup was started | [*metav1.Time](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.27/#time-v1-meta) +`stoppedAt ` | When the backup was terminated | [*metav1.Time](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.27/#time-v1-meta) +`beginWal ` | The starting WAL | string +`endWal ` | The ending WAL | string +`beginLSN ` | The starting xlog | string +`endLSN ` | The ending xlog | string +`error ` | The detected error | string +`commandOutput ` | Unused. Retained for compatibility with old versions. | string +`commandError ` | The backup command output in case of error | string +`instanceID ` | Information to identify the instance where the backup has been taken from | [*InstanceID](#InstanceID) +`snapshotBackupStatus` | BackupSnapshotStatus the status of to the volumeSnapshot backup | [BackupSnapshotStatus](#BackupSnapshotStatus) +`method ` | The backup method being used | BackupMethod @@ -999,6 +1015,7 @@ Name | Description `cluster ` | The cluster to backup | [LocalObjectReference](#LocalObjectReference) `backupOwnerReference` | Indicates which ownerReference should be put inside the created backup resources.
- none: no owner reference for created backup objects (same behavior as before the field was introduced)
- self: sets the Scheduled backup object as owner of the backup
- cluster: set the cluster as owner of the backup
| string `target ` | The policy to decide which instance should perform this backup. If empty, it defaults to `cluster.spec.backup.target`. Available options are empty string, `primary` and `prefer-standby`. `primary` to have backups run always on primary instances, `prefer-standby` to have backups run preferably on the most updated standby, if available. | BackupTarget +`method ` | The backup method to be used, possible options are `barmanObjectStore` and `volumeSnapshot`. Defaults to: `barmanObjectStore`. | BackupMethod @@ -1102,6 +1119,20 @@ Name | Description `nodesUsed ` | NodesUsed represents the count of distinct nodes accommodating the instances. A value of '1' suggests that all instances are hosted on a single node, implying the absence of High Availability (HA). Ideally, this value should be the same as the number of instances in the Postgres HA cluster, implying shared nothing architecture on the compute side. | int32 `successfullyExtracted` | SuccessfullyExtracted indicates if the topology data was extract. It is useful to enact fallback behaviors in synchronous replica election in case of failures | bool + + +## VolumeSnapshotConfiguration + +VolumeSnapshotConfiguration represents the configuration for the execution of snapshot backups. + +Name | Description | Type +---------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------- | ---------------------- +`labels ` | Labels are key-value pairs that will be added to .metadata.labels snapshot resources. | map[string]string +`annotations ` | Annotations key-value pairs that will be added to .metadata.annotations snapshot resources. | map[string]string +`className ` | ClassName specifies the Snapshot Class to be used for PG_DATA PersistentVolumeClaim. It is the default class for the other types if no specific class is present | string +`walClassName ` | WalClassName specifies the Snapshot Class to be used for the PG_WAL PersistentVolumeClaim. | string +`snapshotOwnerReference` | SnapshotOwnerReference indicates the type of owner reference the snapshot should have. . | SnapshotOwnerReference + ## WalBackupConfiguration diff --git a/internal/cmd/plugin/backup/cmd.go b/internal/cmd/plugin/backup/cmd.go index 262deed1f7..d32590a447 100644 --- a/internal/cmd/plugin/backup/cmd.go +++ b/internal/cmd/plugin/backup/cmd.go @@ -24,16 +24,27 @@ import ( "time" "github.com/spf13/cobra" + "golang.org/x/exp/slices" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" apiv1 "github.com/cloudnative-pg/cloudnative-pg/api/v1" "github.com/cloudnative-pg/cloudnative-pg/internal/cmd/plugin" ) +// backupCommandOptions are the options that are provider to the backup +// cnpg command +type backupCommandOptions struct { + backupName string + clusterName string + target apiv1.BackupTarget + method apiv1.BackupMethod +} + // NewCmd creates the new "backup" subcommand func NewCmd() *cobra.Command { var backupName string var backupTarget string + var backupMethod string backupSubcommand := &cobra.Command{ Use: "backup [cluster]", @@ -49,13 +60,34 @@ func NewCmd() *cobra.Command { time.Now().Format("20060102150400")) } - backupTargetPolicy := apiv1.BackupTarget(backupTarget) - switch backupTargetPolicy { - case apiv1.BackupTargetPrimary, apiv1.BackupTargetStandby, "": - return createBackup(cmd.Context(), backupName, clusterName, backupTargetPolicy) - default: + // Check if the backup target is correct + allowedBackupTargets := []string{ + "", + string(apiv1.BackupTargetPrimary), + string(apiv1.BackupTargetStandby), + } + if !slices.Contains(allowedBackupTargets, backupTarget) { return fmt.Errorf("backup-target: %s is not supported by the backup command", backupTarget) } + + // Check if the backup method is correct + allowedBackupMethods := []string{ + "", + string(apiv1.BackupMethodBarmanObjectStore), + string(apiv1.BackupMethodVolumeSnapshot), + } + if !slices.Contains(allowedBackupMethods, backupMethod) { + return fmt.Errorf("backup-method: %s is not supported by the backup command", backupMethod) + } + + return createBackup( + cmd.Context(), + backupCommandOptions{ + backupName: backupName, + clusterName: clusterName, + target: apiv1.BackupTarget(backupTarget), + method: apiv1.BackupMethod(backupMethod), + }) }, } @@ -66,29 +98,39 @@ func NewCmd() *cobra.Command { "The name of the Backup resource that will be created, "+ "defaults to \"[cluster]-[current_timestamp]\"", ) - backupSubcommand.Flags().StringVar( + backupSubcommand.Flags().StringVarP( &backupTarget, "backup-target", + "t", "", "If present, will override the backup target defined in cluster, "+ - "valid value are primary and prefer-standby.", + "valid values are primary and prefer-standby.", + ) + backupSubcommand.Flags().StringVarP( + &backupMethod, + "method", + "m", + "", + "If present, will override the backup method defined in backup resource, "+ + "valid values are volumeSnapshot and barmanObjectStore.", ) return backupSubcommand } // createBackup handles the Backup resource creation -func createBackup(ctx context.Context, backupName, clusterName string, backupTarget apiv1.BackupTarget) error { +func createBackup(ctx context.Context, options backupCommandOptions) error { backup := apiv1.Backup{ ObjectMeta: metav1.ObjectMeta{ Namespace: plugin.Namespace, - Name: backupName, + Name: options.backupName, }, Spec: apiv1.BackupSpec{ Cluster: apiv1.LocalObjectReference{ - Name: clusterName, + Name: options.clusterName, }, - Target: backupTarget, + Target: options.target, + Method: options.method, }, } diff --git a/internal/cmd/plugin/destroy/destroy.go b/internal/cmd/plugin/destroy/destroy.go index 944ffaa53f..2f8c5d4b0f 100644 --- a/internal/cmd/plugin/destroy/destroy.go +++ b/internal/cmd/plugin/destroy/destroy.go @@ -28,7 +28,6 @@ import ( "github.com/cloudnative-pg/cloudnative-pg/controllers" "github.com/cloudnative-pg/cloudnative-pg/internal/cmd/plugin" - "github.com/cloudnative-pg/cloudnative-pg/internal/plugin/resources" "github.com/cloudnative-pg/cloudnative-pg/pkg/reconciler/persistentvolumeclaim" "github.com/cloudnative-pg/cloudnative-pg/pkg/utils" ) @@ -41,7 +40,7 @@ func Destroy(ctx context.Context, clusterName, instanceID string, keepPVC bool) return fmt.Errorf("error deleting instance %s: %v", instanceName, err) } - pvcs, err := resources.GetInstancePVCs(ctx, clusterName, instanceName) + pvcs, err := persistentvolumeclaim.GetInstancePVCs(ctx, plugin.Client, instanceName, plugin.Namespace) if err != nil { return err } diff --git a/internal/cmd/plugin/fence/fence.go b/internal/cmd/plugin/fence/fence.go index 7e4c41d4c2..6359593e91 100644 --- a/internal/cmd/plugin/fence/fence.go +++ b/internal/cmd/plugin/fence/fence.go @@ -21,18 +21,14 @@ import ( "context" "fmt" - v1 "k8s.io/api/core/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "sigs.k8s.io/controller-runtime/pkg/client" - - apiv1 "github.com/cloudnative-pg/cloudnative-pg/api/v1" "github.com/cloudnative-pg/cloudnative-pg/internal/cmd/plugin" + "github.com/cloudnative-pg/cloudnative-pg/pkg/resources" "github.com/cloudnative-pg/cloudnative-pg/pkg/utils" ) // fencingOn marks an instance in a cluster as fenced func fencingOn(ctx context.Context, clusterName string, serverName string) error { - err := ApplyFenceFunc(ctx, plugin.Client, clusterName, plugin.Namespace, serverName, utils.AddFencedInstance) + err := resources.ApplyFenceFunc(ctx, plugin.Client, clusterName, plugin.Namespace, serverName, utils.AddFencedInstance) if err != nil { return err } @@ -42,50 +38,11 @@ func fencingOn(ctx context.Context, clusterName string, serverName string) error // fencingOff marks an instance in a cluster as not fenced func fencingOff(ctx context.Context, clusterName string, serverName string) error { - err := ApplyFenceFunc(ctx, plugin.Client, clusterName, plugin.Namespace, serverName, utils.RemoveFencedInstance) + err := resources.ApplyFenceFunc(ctx, plugin.Client, clusterName, plugin.Namespace, + serverName, utils.RemoveFencedInstance) if err != nil { return err } fmt.Printf("%s unfenced\n", serverName) return nil } - -// ApplyFenceFunc applies a given fencing function to a cluster in a namespace -func ApplyFenceFunc( - ctx context.Context, - cli client.Client, - clusterName string, - namespace string, - serverName string, - fenceFunc func(string, *metav1.ObjectMeta) error, -) error { - var cluster apiv1.Cluster - - // Get the Cluster object - err := cli.Get(ctx, client.ObjectKey{Namespace: namespace, Name: clusterName}, &cluster) - if err != nil { - return err - } - - if serverName != utils.FenceAllServers { - // Check if the Pod exist - var pod v1.Pod - err = cli.Get(ctx, client.ObjectKey{Namespace: namespace, Name: serverName}, &pod) - if err != nil { - return fmt.Errorf("node %s not found in namespace %s", serverName, namespace) - } - } - - fencedCluster := cluster.DeepCopy() - if err = fenceFunc(serverName, &fencedCluster.ObjectMeta); err != nil { - return err - } - fencedCluster.ManagedFields = nil - - err = cli.Patch(ctx, fencedCluster, client.MergeFrom(&cluster)) - if err != nil { - return err - } - - return nil -} diff --git a/internal/cmd/plugin/hibernate/on.go b/internal/cmd/plugin/hibernate/on.go index 4dc0b8da33..1fd0c8ba74 100644 --- a/internal/cmd/plugin/hibernate/on.go +++ b/internal/cmd/plugin/hibernate/on.go @@ -33,10 +33,10 @@ import ( apiv1 "github.com/cloudnative-pg/cloudnative-pg/api/v1" "github.com/cloudnative-pg/cloudnative-pg/internal/cmd/plugin" "github.com/cloudnative-pg/cloudnative-pg/internal/cmd/plugin/destroy" - "github.com/cloudnative-pg/cloudnative-pg/internal/cmd/plugin/fence" - "github.com/cloudnative-pg/cloudnative-pg/internal/plugin/resources" + pluginresources "github.com/cloudnative-pg/cloudnative-pg/internal/plugin/resources" "github.com/cloudnative-pg/cloudnative-pg/pkg/management/log" - pkgres "github.com/cloudnative-pg/cloudnative-pg/pkg/resources" + "github.com/cloudnative-pg/cloudnative-pg/pkg/reconciler/persistentvolumeclaim" + "github.com/cloudnative-pg/cloudnative-pg/pkg/resources" "github.com/cloudnative-pg/cloudnative-pg/pkg/specs" "github.com/cloudnative-pg/cloudnative-pg/pkg/utils" ) @@ -75,7 +75,7 @@ func newOnCommand(ctx context.Context, clusterName string, force bool) (*onComma } // Get the instances to be hibernated - managedInstances, primaryInstance, err := resources.GetInstancePods(ctx, clusterName) + managedInstances, primaryInstance, err := pluginresources.GetInstancePods(ctx, clusterName) if err != nil { return nil, fmt.Errorf("could not get cluster pods: %w", err) } @@ -84,7 +84,7 @@ func newOnCommand(ctx context.Context, clusterName string, force bool) (*onComma } // Get the PVCs that will be hibernated - pvcs, err := resources.GetInstancePVCs(ctx, clusterName, primaryInstance.Name) + pvcs, err := persistentvolumeclaim.GetInstancePVCs(ctx, plugin.Client, primaryInstance.Name, plugin.Namespace) if err != nil { return nil, fmt.Errorf("cannot get PVCs: %w", err) } @@ -178,7 +178,7 @@ func (on *onCommand) fenceClusterStep() error { contextLogger := log.FromContext(on.ctx) contextLogger.Debug("applying the fencing annotation to the cluster manifest") - if err := fence.ApplyFenceFunc( + if err := resources.ApplyFenceFunc( on.ctx, plugin.Client, on.cluster.Name, @@ -203,7 +203,7 @@ func (on *onCommand) rollbackFenceClusterIfNeeded() { contextLogger := log.FromContext(on.ctx) fmt.Println("rolling back hibernation: removing the fencing annotation") - err := fence.ApplyFenceFunc( + err := resources.ApplyFenceFunc( on.ctx, plugin.Client, on.cluster.Name, @@ -219,8 +219,8 @@ func (on *onCommand) rollbackFenceClusterIfNeeded() { // waitInstancesToBeFenced waits for all instances to be shut down func (on *onCommand) waitInstancesToBeFencedStep() error { for _, instance := range on.managedInstances { - if err := retry.OnError(hibernationBackoff, pkgres.RetryAlways, func() error { - running, err := resources.IsInstanceRunning(on.ctx, instance) + if err := retry.OnError(hibernationBackoff, resources.RetryAlways, func() error { + running, err := pluginresources.IsInstanceRunning(on.ctx, instance) if err != nil { return fmt.Errorf("error checking instance status (%v): %w", instance.Name, err) } @@ -298,7 +298,7 @@ func annotatePVCs( pgControlData string, ) error { for _, pvc := range pvcs { - if err := retry.OnError(retry.DefaultBackoff, pkgres.RetryAlways, func() error { + if err := retry.OnError(retry.DefaultBackoff, resources.RetryAlways, func() error { var currentPVC corev1.PersistentVolumeClaim if err := plugin.Client.Get( ctx, @@ -341,7 +341,7 @@ func removePVCannotations( pvcs []corev1.PersistentVolumeClaim, ) error { for _, pvc := range pvcs { - if err := retry.OnError(retry.DefaultBackoff, pkgres.RetryAlways, func() error { + if err := retry.OnError(retry.DefaultBackoff, resources.RetryAlways, func() error { var currentPVC corev1.PersistentVolumeClaim if err := plugin.Client.Get( ctx, diff --git a/internal/cmd/plugin/snapshot/cmd.go b/internal/cmd/plugin/snapshot/cmd.go index 6a89a131f2..fc6583bc11 100644 --- a/internal/cmd/plugin/snapshot/cmd.go +++ b/internal/cmd/plugin/snapshot/cmd.go @@ -19,32 +19,18 @@ package snapshot import ( "context" "fmt" - "time" - storagesnapshotv1 "github.com/kubernetes-csi/external-snapshotter/client/v6/apis/volumesnapshot/v1" "github.com/spf13/cobra" corev1 "k8s.io/api/core/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/util/wait" - "k8s.io/client-go/util/retry" "sigs.k8s.io/controller-runtime/pkg/client" apiv1 "github.com/cloudnative-pg/cloudnative-pg/api/v1" "github.com/cloudnative-pg/cloudnative-pg/internal/cmd/plugin" - "github.com/cloudnative-pg/cloudnative-pg/internal/cmd/plugin/fence" "github.com/cloudnative-pg/cloudnative-pg/internal/plugin/resources" - "github.com/cloudnative-pg/cloudnative-pg/pkg/management/log" - pkgres "github.com/cloudnative-pg/cloudnative-pg/pkg/resources" - "github.com/cloudnative-pg/cloudnative-pg/pkg/utils" + "github.com/cloudnative-pg/cloudnative-pg/pkg/reconciler/persistentvolumeclaim" + "github.com/cloudnative-pg/cloudnative-pg/pkg/utils/snapshot" ) -var snapshotBackoff = wait.Backoff{ - Steps: 4, - Duration: 10 * time.Second, - Factor: 5.0, - Jitter: 0.1, -} - // NewCmd implements the `snapshot` subcommand func NewCmd() *cobra.Command { cmd := &cobra.Command{ @@ -69,12 +55,7 @@ The other replicas will continue working.`, snapshotClassName, _ := cmd.Flags().GetString("volume-snapshot-class-name") snapshotNameSuffix, _ := cmd.Flags().GetString("volume-snapshot-suffix") - snapshotCmd, err := newSnapshotCommand(cmd.Context(), clusterName, snapshotClassName, snapshotNameSuffix) - if err != nil { - return err - } - - return snapshotCmd.execute() + return execute(cmd.Context(), clusterName, snapshotClassName, snapshotNameSuffix) }, } @@ -95,257 +76,59 @@ The other replicas will continue working.`, return cmd } -type snapshotCommand struct { - ctx context.Context - cluster *apiv1.Cluster - targetPod *corev1.Pod - pvcs []corev1.PersistentVolumeClaim - snapshotClassName string - snapshotTime time.Time - snapshotSuffix string -} - -// newSnapshotCommand creates the snapshot command -func newSnapshotCommand( +// execute creates the snapshot command +func execute( ctx context.Context, clusterName string, snapshotClassName string, snapshotSuffix string, -) (*snapshotCommand, error) { - var cluster apiv1.Cluster - - cmd := &snapshotCommand{ - ctx: ctx, - cluster: &cluster, - snapshotClassName: snapshotClassName, - snapshotSuffix: snapshotSuffix, - } - +) error { // Get the Cluster object + var cluster apiv1.Cluster err := plugin.Client.Get( ctx, client.ObjectKey{Namespace: plugin.Namespace, Name: clusterName}, &cluster) if err != nil { - return nil, fmt.Errorf("could not get cluster: %v", err) + return fmt.Errorf("could not get cluster: %v", err) } // Get the target Pod managedInstances, primaryInstance, err := resources.GetInstancePods(ctx, clusterName) if err != nil { - return nil, fmt.Errorf("could not get cluster pods: %w", err) + return fmt.Errorf("could not get cluster pods: %w", err) } if primaryInstance.Name == "" { - return nil, fmt.Errorf("no primary instance found, cannot proceed") + return fmt.Errorf("no primary instance found, cannot proceed") } + var targetPod *corev1.Pod // Get the replica Pod to be fenced for i := len(managedInstances) - 1; i >= 0; i-- { if managedInstances[i].Name != primaryInstance.Name { - cmd.targetPod = managedInstances[i].DeepCopy() + targetPod = managedInstances[i].DeepCopy() break } } - if cmd.targetPod == nil { - return nil, fmt.Errorf("no replicas found, cannot proceed") + if targetPod == nil { + return fmt.Errorf("no replicas found, cannot proceed") } // Get the PVCs that will be snapshotted - cmd.pvcs, err = resources.GetInstancePVCs(ctx, clusterName, cmd.targetPod.Name) - if err != nil { - return nil, fmt.Errorf("cannot get PVCs: %w", err) - } - - return cmd, nil -} - -// execute executes the snapshot command -func (cmd *snapshotCommand) execute() error { - if err := cmd.checkPreconditionsStep(); err != nil { - return err - } - - if err := cmd.fencePodStep(); err != nil { - return err - } - defer cmd.rollbackFencePod() - - if err := cmd.waitPodToBeFencedStep(); err != nil { - return err - } - - if err := cmd.snapshotPVCGroupStep(); err != nil { - return err - } - - return cmd.waitSnapshotToBeReadyStep() -} - -// printAdvancement prints an advancement status on the procedure -func (cmd *snapshotCommand) printAdvancement(msg string, args ...interface{}) { - fmt.Printf(msg, args...) - fmt.Println() -} - -// checkPreconditionsStep checks if the preconditions for the execution of this step are -// met or not. If they are not met, it will return an error -func (cmd *snapshotCommand) checkPreconditionsStep() error { - // We should refuse to hibernate a cluster that was fenced already - fencedInstances, err := utils.GetFencedInstances(cmd.cluster.Annotations) - if err != nil { - return fmt.Errorf("could not check if cluster is fenced: %v", err) - } - - if fencedInstances.Len() > 0 { - return fmt.Errorf("cannot hibernate a cluster that has fenced instances") - } - - return nil -} - -// fencePodStep fence the target Pod -func (cmd *snapshotCommand) fencePodStep() error { - return fence.ApplyFenceFunc( - cmd.ctx, - plugin.Client, - cmd.cluster.Name, - plugin.Namespace, - cmd.targetPod.Name, - utils.AddFencedInstance, - ) -} - -// rollbackFencePod removes the fencing status from the cluster -func (cmd *snapshotCommand) rollbackFencePod() { - contextLogger := log.FromContext(cmd.ctx) - - cmd.printAdvancement("unfencing pod %s", cmd.targetPod.Name) - err := fence.ApplyFenceFunc( - cmd.ctx, - plugin.Client, - cmd.cluster.Name, - plugin.Namespace, - utils.FenceAllServers, - utils.RemoveFencedInstance, - ) + pvcs, err := persistentvolumeclaim.GetInstancePVCs(ctx, plugin.Client, targetPod.Name, plugin.Namespace) if err != nil { - contextLogger.Error( - err, "Rolling back from pod fencing failed", - "targetPod", cmd.targetPod.Name, - ) + return fmt.Errorf("cannot get PVCs: %w", err) } -} -// waitPodToBeFencedStep waits for the target Pod to be shut down -func (cmd *snapshotCommand) waitPodToBeFencedStep() error { - cmd.printAdvancement("waiting for %s to be fenced", cmd.targetPod.Name) - - return retry.OnError(snapshotBackoff, pkgres.RetryAlways, func() error { - running, err := resources.IsInstanceRunning(cmd.ctx, *cmd.targetPod) - if err != nil { - return fmt.Errorf("error checking instance status (%v): %w", cmd.targetPod.Name, err) - } - if running { - return fmt.Errorf("instance still running (%v)", cmd.targetPod.Name) - } - return nil - }) -} - -// snapshotPVCGroup creates a volumeSnapshot resource for every PVC -// used by the Pod -func (cmd *snapshotCommand) snapshotPVCGroupStep() error { - cmd.snapshotTime = time.Now() - - for i := range cmd.pvcs { - if err := cmd.createSnapshot(&cmd.pvcs[i]); err != nil { - return err - } - } - - return nil -} - -// waitSnapshotToBeReadyStep waits for every PVC snapshot to be ready to use -func (cmd *snapshotCommand) waitSnapshotToBeReadyStep() error { - for i := range cmd.pvcs { - name := cmd.getSnapshotName(cmd.pvcs[i].Name) - if err := cmd.waitSnapshot(name); err != nil { - return err - } - } - - return nil -} - -// createSnapshot creates a VolumeSnapshot resource for the given PVC and -// add it to the command status -func (cmd *snapshotCommand) createSnapshot(pvc *corev1.PersistentVolumeClaim) error { - name := cmd.getSnapshotName(pvc.Name) - - var snapshotClassName *string - if cmd.snapshotClassName != "" { - snapshotClassName = &cmd.snapshotClassName - } - - snapshot := storagesnapshotv1.VolumeSnapshot{ - ObjectMeta: metav1.ObjectMeta{ - Name: name, - Namespace: pvc.Namespace, - }, - Spec: storagesnapshotv1.VolumeSnapshotSpec{ - Source: storagesnapshotv1.VolumeSnapshotSource{ - PersistentVolumeClaimName: &pvc.Name, - }, - VolumeSnapshotClassName: snapshotClassName, - }, - } - - err := plugin.Client.Create(cmd.ctx, &snapshot) - if err != nil { - return fmt.Errorf("while creating VolumeSnapshot %s: %w", snapshot.Name, err) - } - - return nil -} - -// waitSnapshot waits for a certain snapshot to be ready to use -func (cmd *snapshotCommand) waitSnapshot(name string) error { - cmd.printAdvancement("waiting for VolumeSnapshot %s to be ready to use", name) - - return retry.OnError(snapshotBackoff, pkgres.RetryAlways, func() error { - var snapshot storagesnapshotv1.VolumeSnapshot - - err := plugin.Client.Get( - cmd.ctx, - client.ObjectKey{ - Namespace: cmd.cluster.Namespace, - Name: name, - }, - &snapshot, - ) - if err != nil { - return fmt.Errorf("snapshot %s is not available: %w", name, err) - } - - if snapshot.Status != nil && snapshot.Status.Error != nil { - return fmt.Errorf("snapshot %s is not ready to use.\nError: %v", name, snapshot.Status.Error.Message) - } - - if snapshot.Status == nil || snapshot.Status.ReadyToUse == nil || !*snapshot.Status.ReadyToUse { - return fmt.Errorf("snapshot %s is not ready to use", name) - } - - return nil - }) -} - -// getSnapshotName gets the snapshot name for a certain PVC -func (cmd *snapshotCommand) getSnapshotName(pvcName string) string { - if cmd.snapshotSuffix != "" { - return fmt.Sprintf("%s-%v", pvcName, cmd.snapshotSuffix) - } + executor := snapshot.NewExecutorBuilder(plugin.Client, apiv1.VolumeSnapshotConfiguration{ + ClassName: snapshotClassName, + SnapshotOwnerReference: "none", + }). + FenceInstance(true). + WithSnapshotSuffix(snapshotSuffix). + Build() - return fmt.Sprintf("%s-%v", pvcName, cmd.snapshotTime.Unix()) + _, err = executor.Execute(ctx, &cluster, targetPod, pvcs) + return err } diff --git a/internal/plugin/resources/instance.go b/internal/plugin/resources/instance.go index d18653daf6..7bc5618652 100644 --- a/internal/plugin/resources/instance.go +++ b/internal/plugin/resources/instance.go @@ -24,8 +24,6 @@ import ( "time" v1 "k8s.io/api/core/v1" - apierrs "k8s.io/apimachinery/pkg/api/errors" - "k8s.io/apimachinery/pkg/types" "k8s.io/client-go/kubernetes" "k8s.io/client-go/rest" "k8s.io/client-go/util/exec" @@ -35,7 +33,6 @@ import ( "github.com/cloudnative-pg/cloudnative-pg/internal/cmd/plugin" "github.com/cloudnative-pg/cloudnative-pg/pkg/management/log" "github.com/cloudnative-pg/cloudnative-pg/pkg/postgres" - "github.com/cloudnative-pg/cloudnative-pg/pkg/reconciler/persistentvolumeclaim" "github.com/cloudnative-pg/cloudnative-pg/pkg/specs" "github.com/cloudnative-pg/cloudnative-pg/pkg/utils" ) @@ -122,60 +119,6 @@ func getReplicaStatusFromPodViaExec( return result } -// GetInstancePVCs gets all the PVC associated with a given instance -func GetInstancePVCs( - ctx context.Context, - clusterName string, - instanceName string, -) ([]v1.PersistentVolumeClaim, error) { - cluster := &corev1.Cluster{} - if err := plugin.Client.Get( - ctx, - types.NamespacedName{ - Name: clusterName, - Namespace: plugin.Namespace, - }, - cluster, - ); err != nil { - return nil, err - } - - var pvcs []v1.PersistentVolumeClaim - - pgDataName := persistentvolumeclaim.GetName(instanceName, utils.PVCRolePgData) - pgData, err := getPVC(ctx, pgDataName) - if err != nil { - return nil, err - } - if pgData != nil { - pvcs = append(pvcs, *pgData) - } - - pgWalName := persistentvolumeclaim.GetName(instanceName, utils.PVCRolePgWal) - pgWal, err := getPVC(ctx, pgWalName) - if err != nil { - return nil, err - } - if pgWal != nil { - pvcs = append(pvcs, *pgWal) - } - - return pvcs, nil -} - -// getPVC returns the pvc if found or any error that isn't apierrs.IsNotFound -func getPVC(ctx context.Context, name string) (*v1.PersistentVolumeClaim, error) { - var pvc v1.PersistentVolumeClaim - err := plugin.Client.Get(ctx, types.NamespacedName{Name: name, Namespace: plugin.Namespace}, &pvc) - if apierrs.IsNotFound(err) { - return nil, nil - } - if err != nil { - return nil, err - } - return &pvc, nil -} - // IsInstanceRunning returns a boolean indicating if the given instance is running and any error encountered func IsInstanceRunning( ctx context.Context, diff --git a/internal/scheme/scheme.go b/internal/scheme/scheme.go index be8385e82d..4a31ed57d9 100644 --- a/internal/scheme/scheme.go +++ b/internal/scheme/scheme.go @@ -17,6 +17,7 @@ limitations under the License. package scheme import ( + storagesnapshotv1 "github.com/kubernetes-csi/external-snapshotter/client/v6/apis/volumesnapshot/v1" monitoringv1 "github.com/prometheus-operator/prometheus-operator/pkg/apis/monitoring/v1" apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" "k8s.io/apimachinery/pkg/runtime" @@ -63,6 +64,13 @@ func (b *Builder) WithAPIExtensionV1() *Builder { return b } +// WithStorageSnapshotV1 adds storagesnapshotv1 +func (b *Builder) WithStorageSnapshotV1() *Builder { + _ = storagesnapshotv1.AddToScheme(b.scheme) + + return b +} + // Build returns the built scheme func (b *Builder) Build() *runtime.Scheme { return b.scheme @@ -75,6 +83,7 @@ func BuildWithAllKnownScheme() *runtime.Scheme { WithClientGoScheme(). WithMonitoringV1(). WithAPIExtensionV1(). + WithStorageSnapshotV1(). Build() // +kubebuilder:scaffold:scheme diff --git a/pkg/management/postgres/backup.go b/pkg/management/postgres/backup.go index 11fb53214e..1a7bea07c1 100644 --- a/pkg/management/postgres/backup.go +++ b/pkg/management/postgres/backup.go @@ -283,13 +283,7 @@ func (b *BackupCommand) run(ctx context.Context) { if failErr := b.retryWithRefreshedCluster(ctx, func() error { origCluster := b.Cluster.DeepCopy() - meta.SetStatusCondition(&b.Cluster. - Status.Conditions, metav1.Condition{ - Type: string(apiv1.ConditionBackup), - Status: metav1.ConditionFalse, - Reason: string(apiv1.ConditionReasonLastBackupFailed), - Message: err.Error(), - }) + meta.SetStatusCondition(&b.Cluster.Status.Conditions, *apiv1.BuildClusterBackupFailedCondition(err)) b.Cluster.Status.LastFailedBackup = utils.GetCurrentTimestampWithFormat(time.RFC3339) return b.Client.Status().Patch(ctx, b.Cluster, client.MergeFrom(origCluster)) @@ -319,12 +313,8 @@ func (b *BackupCommand) takeBackup(ctx context.Context) error { // Update backup status in cluster conditions on startup if err := b.retryWithRefreshedCluster(ctx, func() error { - return conditions.Patch(ctx, b.Client, b.Cluster, &metav1.Condition{ - Type: string(apiv1.ConditionBackup), - Status: metav1.ConditionFalse, - Reason: string(apiv1.ConditionBackupStarted), - Message: "New Backup starting up", - }) + // TODO: this condition is set only here, never removed or handled? + return conditions.Patch(ctx, b.Client, b.Cluster, apiv1.BackupStartingCondition) }); err != nil { b.Log.Error(err, "Error changing backup condition (backup started)") // We do not terminate here because we could still have a good backup @@ -363,12 +353,7 @@ func (b *BackupCommand) takeBackup(ctx context.Context) error { // Update backup status in cluster conditions on backup completion if err := b.retryWithRefreshedCluster(ctx, func() error { - return conditions.Patch(ctx, b.Client, b.Cluster, &metav1.Condition{ - Type: string(apiv1.ConditionBackup), - Status: metav1.ConditionTrue, - Reason: string(apiv1.ConditionReasonLastBackupSucceeded), - Message: "Backup was successful", - }) + return conditions.Patch(ctx, b.Client, b.Cluster, apiv1.BackupSucceededCondition) }); err != nil { b.Log.Error(err, "Can't update the cluster with the completed backup data") } diff --git a/pkg/reconciler/persistentvolumeclaim/resources.go b/pkg/reconciler/persistentvolumeclaim/resources.go index 50fdc25858..87a4ded250 100644 --- a/pkg/reconciler/persistentvolumeclaim/resources.go +++ b/pkg/reconciler/persistentvolumeclaim/resources.go @@ -17,10 +17,14 @@ limitations under the License. package persistentvolumeclaim import ( + "context" "fmt" corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/types" "k8s.io/utils/strings/slices" + "sigs.k8s.io/controller-runtime/pkg/client" apiv1 "github.com/cloudnative-pg/cloudnative-pg/api/v1" "github.com/cloudnative-pg/cloudnative-pg/pkg/utils" @@ -271,3 +275,45 @@ func getStorageSource( return source, nil } + +// GetInstancePVCs gets all the PVC associated with a given instance +func GetInstancePVCs( + ctx context.Context, + cli client.Client, + instanceName string, + namespace string, +) ([]corev1.PersistentVolumeClaim, error) { + getPVC := func(name string) (*corev1.PersistentVolumeClaim, error) { + var pvc corev1.PersistentVolumeClaim + err := cli.Get(ctx, types.NamespacedName{Name: name, Namespace: namespace}, &pvc) + if errors.IsNotFound(err) { + return nil, nil + } + if err != nil { + return nil, err + } + return &pvc, nil + } + + var pvcs []corev1.PersistentVolumeClaim + + pgDataName := GetName(instanceName, utils.PVCRolePgData) + pgData, err := getPVC(pgDataName) + if err != nil { + return nil, err + } + if pgData != nil { + pvcs = append(pvcs, *pgData) + } + + pgWalName := GetName(instanceName, utils.PVCRolePgWal) + pgWal, err := getPVC(pgWalName) + if err != nil { + return nil, err + } + if pgWal != nil { + pvcs = append(pvcs, *pgWal) + } + + return pvcs, nil +} diff --git a/pkg/resources/fencing.go b/pkg/resources/fencing.go new file mode 100644 index 0000000000..2e97265c14 --- /dev/null +++ b/pkg/resources/fencing.go @@ -0,0 +1,68 @@ +/* +Copyright The CloudNativePG Contributors + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package resources + +import ( + "context" + "fmt" + + corev1 "k8s.io/api/core/v1" + v1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "sigs.k8s.io/controller-runtime/pkg/client" + + apiv1 "github.com/cloudnative-pg/cloudnative-pg/api/v1" + "github.com/cloudnative-pg/cloudnative-pg/pkg/utils" +) + +// ApplyFenceFunc applies a given fencing function to a cluster in a namespace +func ApplyFenceFunc( + ctx context.Context, + cli client.Client, + clusterName string, + namespace string, + serverName string, + fenceFunc func(string, *v1.ObjectMeta) error, +) error { + var cluster apiv1.Cluster + + // Get the Cluster object + err := cli.Get(ctx, client.ObjectKey{Namespace: namespace, Name: clusterName}, &cluster) + if err != nil { + return err + } + + if serverName != utils.FenceAllServers { + // Check if the Pod exist + var pod corev1.Pod + err = cli.Get(ctx, client.ObjectKey{Namespace: namespace, Name: serverName}, &pod) + if err != nil { + return fmt.Errorf("node %s not found in namespace %s", serverName, namespace) + } + } + + fencedCluster := cluster.DeepCopy() + if err = fenceFunc(serverName, &fencedCluster.ObjectMeta); err != nil { + return err + } + + err = cli.Patch(ctx, fencedCluster, client.MergeFrom(&cluster)) + if err != nil { + return err + } + + return nil +} diff --git a/pkg/utils/operations.go b/pkg/utils/operations.go index 8495d11818..78ae73afb8 100644 --- a/pkg/utils/operations.go +++ b/pkg/utils/operations.go @@ -42,8 +42,8 @@ func CollectDifferencesFromMaps(p1 map[string]string, p2 map[string]string) map[ return nil } -// isMapSubset returns true if mapSubset is a subset of mapSet otherwise false -func isMapSubset(mapSet map[string]string, mapSubset map[string]string) bool { +// IsMapSubset returns true if mapSubset is a subset of mapSet otherwise false +func IsMapSubset(mapSet map[string]string, mapSubset map[string]string) bool { if len(mapSet) < len(mapSubset) { return false } @@ -86,7 +86,7 @@ func IsLabelSubset( } } - return isMapSubset(mapSet, mapToEvaluate) + return IsMapSubset(mapSet, mapToEvaluate) } // IsAnnotationSubset checks if a collection of annotations is a subset of another @@ -110,5 +110,5 @@ func IsAnnotationSubset( } } - return isMapSubset(mapSet, mapToEvaluate) + return IsMapSubset(mapSet, mapToEvaluate) } diff --git a/pkg/utils/operations_test.go b/pkg/utils/operations_test.go index 3d8883b721..f619946664 100644 --- a/pkg/utils/operations_test.go +++ b/pkg/utils/operations_test.go @@ -42,6 +42,29 @@ var _ = Describe("Difference of values of maps", func() { }) }) +var _ = Describe("Set relationship between maps", func() { + It("An empty map is subset of every possible map", func() { + Expect(IsMapSubset(nil, nil)).To(BeTrue()) + Expect(IsMapSubset(map[string]string{"one": "1"}, nil)).To(BeTrue()) + + Expect(IsMapSubset(nil, map[string]string{"one": "1"})).To(BeFalse()) + }) + + It("Two maps containing different elements are not subsets", func() { + Expect(IsMapSubset(map[string]string{"one": "1"}, map[string]string{"two": "2"})).To(BeFalse()) + Expect(IsMapSubset(map[string]string{"two": "2"}, map[string]string{"one": "1"})).To(BeFalse()) + }) + + It("The subset relationship is not invertible", func() { + Expect(IsMapSubset(map[string]string{"one": "1", "two": "2"}, map[string]string{"two": "2"})).To(BeTrue()) + Expect(IsMapSubset(map[string]string{"two": "2"}, map[string]string{"one": "1", "two": "2"})).To(BeFalse()) + }) + + It("Two equal maps are subsets", func() { + Expect(IsMapSubset(map[string]string{"one": "1"}, map[string]string{"one": "1"})).To(BeTrue()) + }) +}) + var _ = Describe("Testing Annotations and labels subset", func() { const environment = "environment" const department = "finance" diff --git a/pkg/utils/snapshot/doc.go b/pkg/utils/snapshot/doc.go new file mode 100644 index 0000000000..dee9c208db --- /dev/null +++ b/pkg/utils/snapshot/doc.go @@ -0,0 +1,18 @@ +/* +Copyright The CloudNativePG Contributors + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Package snapshot contains reusable snapshot execution logic +package snapshot diff --git a/pkg/utils/snapshot/executor.go b/pkg/utils/snapshot/executor.go new file mode 100644 index 0000000000..1f500b3c30 --- /dev/null +++ b/pkg/utils/snapshot/executor.go @@ -0,0 +1,348 @@ +/* +Copyright The CloudNativePG Contributors + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package snapshot + +import ( + "context" + "errors" + "fmt" + "time" + + storagesnapshotv1 "github.com/kubernetes-csi/external-snapshotter/client/v6/apis/volumesnapshot/v1" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + "k8s.io/apimachinery/pkg/util/wait" + "k8s.io/client-go/util/retry" + "sigs.k8s.io/controller-runtime/pkg/client" + + apiv1 "github.com/cloudnative-pg/cloudnative-pg/api/v1" + "github.com/cloudnative-pg/cloudnative-pg/pkg/management/log" + "github.com/cloudnative-pg/cloudnative-pg/pkg/resources" + "github.com/cloudnative-pg/cloudnative-pg/pkg/utils" +) + +var snapshotBackoff = wait.Backoff{ + Steps: 7, + Duration: 10 * time.Second, + Factor: 5.0, + Jitter: 0.1, +} + +// Executor is an object capable of executing a volume snapshot on a running cluster +type Executor struct { + cli client.Client + shouldFence bool + snapshotSuffix string + printAdvancementFunc func(msg string) + snapshotEnrichFunc func(vs *storagesnapshotv1.VolumeSnapshot) + snapshotConfig apiv1.VolumeSnapshotConfiguration +} + +// ExecutorBuilder is a struct capable of creating an Executor +type ExecutorBuilder struct { + executor Executor +} + +// NewExecutorBuilder instantiates a new ExecutorBuilder with the minimum required data +func NewExecutorBuilder(cli client.Client, config apiv1.VolumeSnapshotConfiguration) *ExecutorBuilder { + return &ExecutorBuilder{ + executor: Executor{ + cli: cli, + snapshotEnrichFunc: func(vs *storagesnapshotv1.VolumeSnapshot) {}, + snapshotConfig: config, + }, + } +} + +// FenceInstance instructs if the Executor should fence or not the instance while taking the snapshot +func (e *ExecutorBuilder) FenceInstance(fence bool) *ExecutorBuilder { + e.executor.shouldFence = fence + return e +} + +// WithSnapshotEnrich accepts a function capable of adding new data to the storagesnapshotv1.VolumeSnapshot resource +func (e *ExecutorBuilder) WithSnapshotEnrich(enrich func(vs *storagesnapshotv1.VolumeSnapshot)) *ExecutorBuilder { + e.executor.snapshotEnrichFunc = enrich + return e +} + +// WithPrintLogger sets a Println type of logging +func (e *ExecutorBuilder) WithPrintLogger() *ExecutorBuilder { + e.executor.printAdvancementFunc = func(msg string) { + fmt.Println(msg) + } + + return e +} + +// WithSnapshotSuffix the suffix that should be added to the snapshots. Defaults to unix timestamp. +func (e *ExecutorBuilder) WithSnapshotSuffix(suffix string) *ExecutorBuilder { + e.executor.snapshotSuffix = suffix + return e +} + +// Build returns the Executor instance +func (e *ExecutorBuilder) Build() *Executor { + return &e.executor +} + +func (se *Executor) ensureLoggerIsPresent(ctx context.Context) { + if se.printAdvancementFunc != nil { + return + } + + contextLogger := log.FromContext(ctx) + se.printAdvancementFunc = func(msg string) { + contextLogger.Info(msg) + } +} + +// Execute the volume snapshot of the given cluster instance +func (se *Executor) Execute( + ctx context.Context, + cluster *apiv1.Cluster, + targetPod *corev1.Pod, + pvcs []corev1.PersistentVolumeClaim, +) ([]*storagesnapshotv1.VolumeSnapshot, error) { + se.ensureLoggerIsPresent(ctx) + + if err := se.checkPreconditionsStep(cluster); err != nil { + return nil, err + } + + if se.shouldFence { + if err := se.fencePodStep(ctx, cluster, targetPod); err != nil { + return nil, err + } + defer se.rollbackFencePod(ctx, cluster, targetPod) + + if err := se.waitPodToBeFencedStep(ctx, targetPod); err != nil { + return nil, err + } + } + + // if we have no suffix specified from the user we use unix timestamp + if se.snapshotSuffix == "" { + se.snapshotSuffix = fmt.Sprintf("%d", time.Now().Unix()) + } + + snapshots, err := se.snapshotPVCGroupStep(ctx, pvcs) + if err != nil { + return nil, err + } + + return snapshots, se.waitSnapshotToBeReadyStep(ctx, pvcs) +} + +// checkPreconditionsStep checks if the preconditions for the execution of this step are +// met or not. If they are not met, it will return an error +func (se *Executor) checkPreconditionsStep( + cluster *apiv1.Cluster, +) error { + se.printAdvancementFunc("ensuring that no pod is fenced before starting") + + fencedInstances, err := utils.GetFencedInstances(cluster.Annotations) + if err != nil { + return fmt.Errorf("could not check if cluster is fenced: %v", err) + } + + if fencedInstances.Len() > 0 { + return errors.New("cannot execute volume snapshot on a cluster that has fenced instances") + } + + return nil +} + +// fencePodStep fence the target Pod +func (se *Executor) fencePodStep( + ctx context.Context, + cluster *apiv1.Cluster, + targetPod *corev1.Pod, +) error { + se.printAdvancementFunc(fmt.Sprintf("fencing pod: %s", targetPod.Namespace)) + return resources.ApplyFenceFunc( + ctx, + se.cli, + cluster.Name, + cluster.Namespace, + targetPod.Name, + utils.AddFencedInstance, + ) +} + +// rollbackFencePod removes the fencing status from the cluster +func (se *Executor) rollbackFencePod( + ctx context.Context, + cluster *apiv1.Cluster, + targetPod *corev1.Pod, +) { + contextLogger := log.FromContext(ctx) + + se.printAdvancementFunc(fmt.Sprintf("unfencing pod %s", targetPod.Name)) + if err := resources.ApplyFenceFunc( + ctx, + se.cli, + cluster.Name, + cluster.Namespace, + utils.FenceAllServers, + utils.RemoveFencedInstance, + ); err != nil { + contextLogger.Error( + err, "while rolling back the pod from the fencing state", + "targetPod", targetPod.Name, + ) + } +} + +// waitPodToBeFencedStep waits for the target Pod to be shut down +func (se *Executor) waitPodToBeFencedStep( + ctx context.Context, + targetPod *corev1.Pod, +) error { + se.printAdvancementFunc(fmt.Sprintf("waiting for %s to be fenced", targetPod.Name)) + + return retry.OnError(snapshotBackoff, resources.RetryAlways, func() error { + var pod corev1.Pod + err := se.cli.Get(ctx, types.NamespacedName{Name: targetPod.Name, Namespace: targetPod.Namespace}, &pod) + if err != nil { + return err + } + ready := utils.IsPodReady(pod) + if ready { + return fmt.Errorf("instance still running (%v)", targetPod.Name) + } + return nil + }) +} + +// snapshotPVCGroup creates a volumeSnapshot resource for every PVC +// used by the Pod +func (se *Executor) snapshotPVCGroupStep( + ctx context.Context, + pvcs []corev1.PersistentVolumeClaim, +) ([]*storagesnapshotv1.VolumeSnapshot, error) { + createdSnapshots := make([]*storagesnapshotv1.VolumeSnapshot, len(pvcs)) + for i := range pvcs { + snapshot, err := se.createSnapshot(ctx, &pvcs[i]) + if err != nil { + return nil, err + } + createdSnapshots[i] = snapshot + } + + return createdSnapshots, nil +} + +// waitSnapshotToBeReadyStep waits for every PVC snapshot to be ready to use +func (se *Executor) waitSnapshotToBeReadyStep( + ctx context.Context, + pvcs []corev1.PersistentVolumeClaim, +) error { + for i := range pvcs { + name := se.getSnapshotName(pvcs[i].Name) + if err := se.waitSnapshot(ctx, name, pvcs[i].Namespace); err != nil { + return err + } + } + + return nil +} + +// createSnapshot creates a VolumeSnapshot resource for the given PVC and +// add it to the command status +func (se *Executor) createSnapshot( + ctx context.Context, + pvc *corev1.PersistentVolumeClaim, +) (*storagesnapshotv1.VolumeSnapshot, error) { + name := se.getSnapshotName(pvc.Name) + var snapshotClassName *string + role := utils.PVCRole(pvc.Labels[utils.PvcRoleLabelName]) + if role == utils.PVCRolePgWal && se.snapshotConfig.WalClassName != "" { + snapshotClassName = &se.snapshotConfig.WalClassName + } + + // this is the default value if nothing else was assigned + if snapshotClassName == nil && se.snapshotConfig.ClassName != "" { + snapshotClassName = &se.snapshotConfig.ClassName + } + + labels := pvc.Labels + utils.MergeMap(labels, se.snapshotConfig.Labels) + annotations := pvc.Annotations + utils.MergeMap(annotations, se.snapshotConfig.Annotations) + + snapshot := storagesnapshotv1.VolumeSnapshot{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: pvc.Namespace, + Labels: labels, + Annotations: annotations, + }, + Spec: storagesnapshotv1.VolumeSnapshotSpec{ + Source: storagesnapshotv1.VolumeSnapshotSource{ + PersistentVolumeClaimName: &pvc.Name, + }, + VolumeSnapshotClassName: snapshotClassName, + }, + } + + se.snapshotEnrichFunc(&snapshot) + + err := se.cli.Create(ctx, &snapshot) + if err != nil { + return nil, fmt.Errorf("while creating VolumeSnapshot %s: %w", snapshot.Name, err) + } + + return &snapshot, nil +} + +// waitSnapshot waits for a certain snapshot to be ready to use +func (se *Executor) waitSnapshot(ctx context.Context, name, namespace string) error { + se.printAdvancementFunc(fmt.Sprintf("waiting for VolumeSnapshot %s to be ready to use", name)) + + return retry.OnError(snapshotBackoff, resources.RetryAlways, func() error { + var snapshot storagesnapshotv1.VolumeSnapshot + + err := se.cli.Get( + ctx, + client.ObjectKey{ + Namespace: namespace, + Name: name, + }, + &snapshot, + ) + if err != nil { + return fmt.Errorf("snapshot %s is not available: %w", name, err) + } + + if snapshot.Status != nil && snapshot.Status.Error != nil { + return fmt.Errorf("snapshot %s is not ready to use.\nError: %v", name, snapshot.Status.Error.Message) + } + + if snapshot.Status == nil || snapshot.Status.ReadyToUse == nil || !*snapshot.Status.ReadyToUse { + return fmt.Errorf("snapshot %s is not ready to use", name) + } + + return nil + }) +} + +// getSnapshotName gets the snapshot name for a certain PVC +func (se *Executor) getSnapshotName(pvcName string) string { + return fmt.Sprintf("%s-%s", pvcName, se.snapshotSuffix) +} diff --git a/tests/e2e/fixtures/volume_snapshot/cluster-pvc-snapshot.yaml.template b/tests/e2e/fixtures/volume_snapshot/cluster-pvc-snapshot.yaml.template index 3ac230ade4..252aa63800 100644 --- a/tests/e2e/fixtures/volume_snapshot/cluster-pvc-snapshot.yaml.template +++ b/tests/e2e/fixtures/volume_snapshot/cluster-pvc-snapshot.yaml.template @@ -15,7 +15,6 @@ spec: size: 1Gi backup: - target: primary barmanObjectStore: destinationPath: s3://cluster-backups/ endpointURL: https://minio-service:9000 diff --git a/tests/e2e/fixtures/volume_snapshot/declarative-backup-cluster-restore.yaml.template b/tests/e2e/fixtures/volume_snapshot/declarative-backup-cluster-restore.yaml.template new file mode 100644 index 0000000000..bc19a2c21b --- /dev/null +++ b/tests/e2e/fixtures/volume_snapshot/declarative-backup-cluster-restore.yaml.template @@ -0,0 +1,27 @@ +apiVersion: postgresql.cnpg.io/v1 +kind: Cluster +metadata: + name: cluster-declarative-backup-restore +spec: + instances: 1 + primaryUpdateStrategy: unsupervised + + # Persistent storage configuration + storage: + storageClass: ${E2E_CSI_STORAGE_CLASS} + size: 1Gi + walStorage: + storageClass: ${E2E_CSI_STORAGE_CLASS} + size: 1Gi + + bootstrap: + recovery: + volumeSnapshots: + storage: + name: ${SNAPSHOT_NAME_PGDATA} + kind: VolumeSnapshot + apiGroup: snapshot.storage.k8s.io + walStorage: + name: ${SNAPSHOT_NAME_PGWAL} + kind: VolumeSnapshot + apiGroup: snapshot.storage.k8s.io diff --git a/tests/e2e/fixtures/volume_snapshot/declarative-backup-cluster.yaml.template b/tests/e2e/fixtures/volume_snapshot/declarative-backup-cluster.yaml.template new file mode 100644 index 0000000000..194b207088 --- /dev/null +++ b/tests/e2e/fixtures/volume_snapshot/declarative-backup-cluster.yaml.template @@ -0,0 +1,24 @@ +apiVersion: postgresql.cnpg.io/v1 +kind: Cluster +metadata: + name: cluster-declarative-backup +spec: + instances: 2 + primaryUpdateStrategy: unsupervised + + # Persistent storage configuration + storage: + storageClass: ${E2E_CSI_STORAGE_CLASS} + size: 1Gi + walStorage: + storageClass: ${E2E_CSI_STORAGE_CLASS} + size: 1Gi + + backup: + volumeSnapshot: + className: ${E2E_DEFAULT_VOLUMESNAPSHOT_CLASS} + snapshotOwnerReference: cluster + annotations: + test-annotation: test + labels: + test-labels: test diff --git a/tests/e2e/fixtures/volume_snapshot/declarative-backup.yaml.template b/tests/e2e/fixtures/volume_snapshot/declarative-backup.yaml.template new file mode 100644 index 0000000000..2572976745 --- /dev/null +++ b/tests/e2e/fixtures/volume_snapshot/declarative-backup.yaml.template @@ -0,0 +1,8 @@ +apiVersion: postgresql.cnpg.io/v1 +kind: Backup +metadata: + name: cluster-declarative-backup +spec: + method: volumeSnapshot + cluster: + name: cluster-declarative-backup diff --git a/tests/e2e/volume_snapshot_test.go b/tests/e2e/volume_snapshot_test.go index 51cb7e197e..1d0327579e 100644 --- a/tests/e2e/volume_snapshot_test.go +++ b/tests/e2e/volume_snapshot_test.go @@ -17,10 +17,17 @@ limitations under the License. package e2e import ( + "fmt" "os" "strings" + volumesnapshot "github.com/kubernetes-csi/external-snapshotter/client/v6/apis/volumesnapshot/v1" + "k8s.io/apimachinery/pkg/types" + k8client "sigs.k8s.io/controller-runtime/pkg/client" + + apiv1 "github.com/cloudnative-pg/cloudnative-pg/api/v1" "github.com/cloudnative-pg/cloudnative-pg/pkg/certs" + "github.com/cloudnative-pg/cloudnative-pg/pkg/utils" "github.com/cloudnative-pg/cloudnative-pg/tests" testUtils "github.com/cloudnative-pg/cloudnative-pg/tests/utils" @@ -257,4 +264,128 @@ var _ = Describe("Verify Volume Snapshot", }) }) }) + + Context("Declarative Volume Snapshot", Ordered, func() { + // test env constants + const ( + namespacePrefix = "declarative-snapshot-backup" + level = tests.High + filesDir = fixturesDir + "/volume_snapshot" + ) + // file constants + const ( + clusterToBackupFilePath = filesDir + "/declarative-backup-cluster.yaml.template" + clusterToRestoreFilePath = filesDir + "/declarative-backup-cluster-restore.yaml.template" + backupFileFilePath = filesDir + "/declarative-backup.yaml.template" + ) + + // database constants + const ( + tableName = "test" + ) + + BeforeAll(func() { + if testLevelEnv.Depth < int(level) { + Skip("Test depth is lower than the amount requested for this test") + } + + if !(IsLocal() || IsGKE()) { + Skip("This test is only executed on gke, openshift and local") + } + + var err error + namespace, err = env.CreateUniqueNamespace(namespacePrefix) + Expect(err).ToNot(HaveOccurred()) + DeferCleanup(func() error { + if CurrentSpecReport().Failed() { + env.DumpNamespaceObjects(namespace, "out/"+CurrentSpecReport().LeafNodeText+".log") + } + _ = os.Unsetenv("SNAPSHOT_NAME_PGDATA") + _ = os.Unsetenv("SNAPSHOT_NAME_PGWAL") + return env.DeleteNamespace(namespace) + }) + }) + + It("creating a declarative cold backup and restoring it", func() { + clusterToBackupName, err := env.GetResourceNameFromYAML(clusterToBackupFilePath) + Expect(err).ToNot(HaveOccurred()) + + By("creating the cluster on which to execute the backup", func() { + AssertCreateCluster(namespace, clusterToBackupName, clusterToBackupFilePath, env) + }) + + By("inserting test data", func() { + AssertCreateTestData(namespace, clusterToBackupName, tableName, psqlClientPod) + }) + + backupName, err := env.GetResourceNameFromYAML(backupFileFilePath) + Expect(err).ToNot(HaveOccurred()) + + By("executing the backup", func() { + err := CreateResourcesFromFileWithError(namespace, backupFileFilePath) + Expect(err).ToNot(HaveOccurred()) + }) + + var backup apiv1.Backup + By("waiting the backup to complete", func() { + Eventually(func(g Gomega) { + err := env.Client.Get(env.Ctx, types.NamespacedName{Name: backupName, Namespace: namespace}, &backup) + g.Expect(err).ToNot(HaveOccurred()) + g.Expect(backup.Status.Phase).To(BeEquivalentTo(apiv1.BackupPhaseCompleted)) + }, 300).Should(Succeed()) + AssertBackupConditionInClusterStatus(namespace, clusterToBackupName) + }) + + var clusterToBackup *apiv1.Cluster + + By("fetching the created cluster", func() { + clusterToBackup, err = env.GetCluster(namespace, clusterToBackupName) + Expect(err).ToNot(HaveOccurred()) + }) + + snapshotList := volumesnapshot.VolumeSnapshotList{} + By("fetching the volume snapshots", func() { + err := env.Client.List(env.Ctx, &snapshotList, k8client.MatchingLabels{ + utils.ClusterLabelName: clusterToBackupName, + }) + Expect(err).ToNot(HaveOccurred()) + Expect(snapshotList.Items).To(HaveLen(len(backup.Status.BackupSnapshotStatus.Snapshots))) + }) + + By("ensuring that the additional labels and annotations are present", func() { + for _, item := range snapshotList.Items { + snapshotConfig := clusterToBackup.Spec.Backup.VolumeSnapshot + Expect(utils.IsMapSubset(item.Annotations, snapshotConfig.Annotations)).To(BeTrue()) + Expect(utils.IsMapSubset(item.Labels, snapshotConfig.Labels)).To(BeTrue()) + } + }) + + By("setting the snapshot name env variable", func() { + for _, item := range snapshotList.Items { + switch utils.PVCRole(item.Labels[utils.PvcRoleLabelName]) { + case utils.PVCRolePgData: + err = os.Setenv("SNAPSHOT_NAME_PGDATA", item.Name) + case utils.PVCRolePgWal: + err = os.Setenv("SNAPSHOT_NAME_PGWAL", item.Name) + default: + Fail(fmt.Sprintf("Unrecognized PVC snapshot role: %s, name: %s", + item.Labels[utils.PvcRoleLabelName], + item.Name, + )) + } + } + }) + + clusterToRestoreName, err := env.GetResourceNameFromYAML(clusterToRestoreFilePath) + Expect(err).ToNot(HaveOccurred()) + + By("executing the restore", func() { + AssertCreateCluster(namespace, clusterToRestoreName, clusterToRestoreFilePath, env) + }) + + By("checking that the data is present on the restored cluster", func() { + AssertDataExpectedCount(namespace, clusterToRestoreName, tableName, 2, psqlClientPod) + }) + }) + }) }) diff --git a/tests/utils/fence.go b/tests/utils/fence.go index 0206f3a1e3..39af0df6c9 100644 --- a/tests/utils/fence.go +++ b/tests/utils/fence.go @@ -19,7 +19,7 @@ package utils import ( "fmt" - "github.com/cloudnative-pg/cloudnative-pg/internal/cmd/plugin/fence" + "github.com/cloudnative-pg/cloudnative-pg/pkg/resources" "github.com/cloudnative-pg/cloudnative-pg/pkg/utils" ) @@ -49,7 +49,7 @@ func FencingOn( return err } case UsingAnnotation: - err := fence.ApplyFenceFunc(env.Ctx, env.Client, clusterName, namespace, serverName, utils.AddFencedInstance) + err := resources.ApplyFenceFunc(env.Ctx, env.Client, clusterName, namespace, serverName, utils.AddFencedInstance) if err != nil { return err } @@ -75,7 +75,7 @@ func FencingOff( return err } case UsingAnnotation: - err := fence.ApplyFenceFunc(env.Ctx, env.Client, clusterName, namespace, serverName, utils.RemoveFencedInstance) + err := resources.ApplyFenceFunc(env.Ctx, env.Client, clusterName, namespace, serverName, utils.RemoveFencedInstance) if err != nil { return err }