fix: check EMQX condition before revert sts/rs

Signed-off-by: Rory Z <[email protected]>
emqx · Jan 7, 2024 · fa055b6 · fa055b6
1 parent 7fc4bbf
commit fa055b6
Show file tree

Hide file tree

Showing 2 changed files with 18 additions and 14 deletions.
diff --git a/controllers/apps/v2beta1/add_emqx_core.go b/controllers/apps/v2beta1/add_emqx_core.go
@@ -50,14 +50,16 @@ func (a *addCore) reconcile(ctx context.Context, instance *appsv2beta1.EMQX, _ i
 		_ = ctrl.SetControllerReference(instance, preSts, a.Scheme)
 		if err := a.Handler.Create(preSts); err != nil {
 			if k8sErrors.IsAlreadyExists(emperror.Cause(err)) {
-				// Sometimes the updated statefulSet will not be ready, because the EMQX node can not be started.
-				// And then we will rollback EMQX CR spec, the EMQX operator controller will create a new statefulSet.
-				// But the new statefulSet will be the same as the previous one, so we didn't need to create it, just change the EMQX status.
-				if preStsHash == instance.Status.CoreNodesStatus.CurrentRevision {
-					_ = a.updateEMQXStatus(ctx, instance, "RevertStatefulSet", "Revert to current statefulSet", preStsHash)
-					return subResult{}
+				cond := instance.Status.GetLastTrueCondition()
+				if cond != nil && cond.Type != appsv2beta1.Available && cond.Type != appsv2beta1.Ready {
+					// Sometimes the updated statefulSet will not be ready, because the EMQX node can not be started.
+					// And then we will rollback EMQX CR spec, the EMQX operator controller will create a new statefulSet.
+					// But the new statefulSet will be the same as the previous one, so we didn't need to create it, just change the EMQX status.
+					if preStsHash == instance.Status.CoreNodesStatus.CurrentRevision {
+						_ = a.updateEMQXStatus(ctx, instance, "RevertStatefulSet", "Revert to current statefulSet", preStsHash)
+						return subResult{}
+					}
 				}
-
 				if instance.Status.CoreNodesStatus.CollisionCount == nil {
 					instance.Status.CoreNodesStatus.CollisionCount = pointer.Int32(0)
 				}

diff --git a/controllers/apps/v2beta1/add_emqx_repl.go b/controllers/apps/v2beta1/add_emqx_repl.go
@@ -57,14 +57,16 @@ func (a *addRepl) reconcile(ctx context.Context, instance *appsv2beta1.EMQX, _ i
 		_ = ctrl.SetControllerReference(instance, preRs, a.Scheme)
 		if err := a.Handler.Create(preRs); err != nil {
 			if k8sErrors.IsAlreadyExists(emperror.Cause(err)) {
-				// Sometimes the updated replicaSet will not be ready, because the EMQX node can not be started.
-				// And then we will rollback EMQX CR spec, the EMQX operator controller will create a new replicaSet.
-				// But the new replicaSet will be the same as the previous one, so we didn't need to create it, just change the EMQX status.
-				if preRsHash == instance.Status.ReplicantNodesStatus.CurrentRevision {
-					_ = a.updateEMQXStatus(ctx, instance, "RevertReplicaSet", "Revert to current replicaSet", preRsHash)
-					return subResult{}
+				cond := instance.Status.GetLastTrueCondition()
+				if cond != nil && cond.Type != appsv2beta1.Available && cond.Type != appsv2beta1.Ready {
+					// Sometimes the updated replicaSet will not be ready, because the EMQX node can not be started.
+					// And then we will rollback EMQX CR spec, the EMQX operator controller will create a new replicaSet.
+					// But the new replicaSet will be the same as the previous one, so we didn't need to create it, just change the EMQX status.
+					if preRsHash == instance.Status.ReplicantNodesStatus.CurrentRevision {
+						_ = a.updateEMQXStatus(ctx, instance, "RevertReplicaSet", "Revert to current replicaSet", preRsHash)
+						return subResult{}
+					}
 				}
-
 				if instance.Status.ReplicantNodesStatus.CollisionCount == nil {
 					instance.Status.ReplicantNodesStatus.CollisionCount = pointer.Int32(0)
 				}