fix(v2beta1): fix error for when delete pods, emqx still ready

emqx · Aug 10, 2023 · 9f299ef · 9f299ef
1 parent 286ea42
commit 9f299ef
Show file tree

Hide file tree

Showing 3 changed files with 53 additions and 26 deletions.
diff --git a/controllers/apps/v2beta1/emqx_controller.go b/controllers/apps/v2beta1/emqx_controller.go
@@ -87,7 +87,7 @@ func NewEMQXReconciler(mgr manager.Manager) *EMQXReconciler {
 // For more details, check Reconcile and its Result here:
 // - https://pkg.go.dev/sigs.k8s.io/[email protected]/pkg/reconcile
 func (r *EMQXReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) {
-	_ = log.FromContext(ctx)
+	logger := log.FromContext(ctx)
 
 	instance := &appsv2beta1.EMQX{}
 	if err := r.Client.Get(ctx, req.NamespacedName, instance); err != nil {
@@ -117,15 +117,16 @@ func (r *EMQXReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.
 		&syncConfig{r},
 		&addSvc{r},
 		&updatePodConditions{r},
-		&syncPods{r},
 		&updateStatus{r},
+		&syncPods{r},
 	} {
 		subResult := subReconciler.reconcile(ctx, instance, requester)
 		if !subResult.result.IsZero() {
 			return subResult.result, nil
 		}
 		if subResult.err != nil {
 			if innerErr.IsCommonError(subResult.err) {
+				logger.V(1).Info("requeue reconcile", "reconciler", subReconciler, "reason", subResult.err)
 				return ctrl.Result{RequeueAfter: time.Second}, nil
 			}
 			r.EventRecorder.Event(instance, corev1.EventTypeWarning, "ReconcilerFailed", emperror.Cause(subResult.err).Error())

diff --git a/controllers/apps/v2beta1/status_machine.go b/controllers/apps/v2beta1/status_machine.go
@@ -113,14 +113,14 @@ type initializedStatus struct {
 }
 
 func (s *initializedStatus) nextStatus() {
-	s.emqxStatusMachine.emqx.Status.RemoveCondition(appsv2beta1.ReplicantNodesProgressing)
+	s.emqxStatusMachine.emqx.Status.RemoveCondition(appsv2beta1.Ready)
+	s.emqxStatusMachine.emqx.Status.RemoveCondition(appsv2beta1.Available)
+
 	s.emqxStatusMachine.emqx.Status.RemoveCondition(appsv2beta1.ReplicantNodesReady)
+	s.emqxStatusMachine.emqx.Status.RemoveCondition(appsv2beta1.ReplicantNodesProgressing)
 
-	s.emqxStatusMachine.emqx.Status.RemoveCondition(appsv2beta1.CoreNodesProgressing)
 	s.emqxStatusMachine.emqx.Status.RemoveCondition(appsv2beta1.CoreNodesReady)
-
-	s.emqxStatusMachine.emqx.Status.RemoveCondition(appsv2beta1.Available)
-	s.emqxStatusMachine.emqx.Status.RemoveCondition(appsv2beta1.Ready)
+	s.emqxStatusMachine.emqx.Status.RemoveCondition(appsv2beta1.CoreNodesProgressing)
 
 	s.emqxStatusMachine.emqx.Status.SetCondition(metav1.Condition{
 		Type:    appsv2beta1.CoreNodesProgressing,
@@ -254,4 +254,29 @@ type readyStatus struct {
 	emqxStatusMachine *emqxStatusMachine
 }
 
-func (s *readyStatus) nextStatus() {}
+func (s *readyStatus) nextStatus() {
+	emqx := s.emqxStatusMachine.GetEMQX()
+	updateSts, _, _ := getStateFulSetList(context.Background(), s.emqxStatusMachine.client, emqx)
+	if updateSts != nil && updateSts.Status.ReadyReplicas != emqx.Status.CoreNodesStatus.Replicas {
+		s.emqxStatusMachine.initialized.nextStatus()
+		return
+	}
+
+	if appsv2beta1.IsExistReplicant(emqx) {
+		updateRs, _, _ := getReplicaSetList(context.Background(), s.emqxStatusMachine.client, emqx)
+		if updateRs != nil && updateRs.Status.ReadyReplicas != emqx.Status.ReplicantNodesStatus.Replicas {
+			s.emqxStatusMachine.emqx.Status.RemoveCondition(appsv2beta1.Ready)
+			s.emqxStatusMachine.emqx.Status.RemoveCondition(appsv2beta1.Available)
+			s.emqxStatusMachine.emqx.Status.RemoveCondition(appsv2beta1.ReplicantNodesReady)
+
+			emqx.Status.SetCondition(metav1.Condition{
+				Type:    appsv2beta1.ReplicantNodesProgressing,
+				Status:  metav1.ConditionTrue,
+				Reason:  appsv2beta1.ReplicantNodesProgressing,
+				Message: "Replicant nodes progressing",
+			})
+			s.emqxStatusMachine.setCurrentStatus(emqx)
+			return
+		}
+	}
+}
diff --git a/controllers/apps/v2beta1/sync_emqx_config.go b/controllers/apps/v2beta1/sync_emqx_config.go
@@ -28,24 +28,6 @@ func (s *syncConfig) reconcile(ctx context.Context, instance *appsv2beta1.EMQX,
 	}
 
 	lastConfig, ok := instance.Annotations[appsv2beta1.LastEMQXConfigAnnotationKey]
-	if ok && instance.Spec.Config.Data != lastConfig {
-		// Delete readonly configs
-		config, _ := hocon.ParseString(instance.Spec.Config.Data)
-		configObj := config.GetRoot().(hocon.Object)
-		delete(configObj, "node")
-		delete(configObj, "cluster")
-		delete(configObj, "dashboard")
-
-		if err := putEMQXConfigsByAPI(r, instance.Spec.Config.Mode, configObj.String()); err != nil {
-			return subResult{err: emperror.Wrap(err, "failed to put emqx config")}
-		}
-
-		instance.Annotations[appsv2beta1.LastEMQXConfigAnnotationKey] = instance.Spec.Config.Data
-		if err := s.Client.Update(ctx, instance); err != nil {
-			return subResult{err: emperror.Wrap(err, "failed to update emqx instance")}
-		}
-	}
-
 	if !ok {
 		// If it is the first time to start and Mode = Replace, update the EMQX configuration once.
 		if instance.Spec.Config.Mode == "Replace" {
@@ -67,6 +49,25 @@ func (s *syncConfig) reconcile(ctx context.Context, instance *appsv2beta1.EMQX,
 		if err := s.Client.Update(ctx, instance); err != nil {
 			return subResult{err: emperror.Wrap(err, "failed to update emqx instance")}
 		}
+		return subResult{}
+	}
+	if ok && instance.Spec.Config.Data != lastConfig {
+		// Delete readonly configs
+		config, _ := hocon.ParseString(instance.Spec.Config.Data)
+		configObj := config.GetRoot().(hocon.Object)
+		delete(configObj, "node")
+		delete(configObj, "cluster")
+		delete(configObj, "dashboard")
+
+		if err := putEMQXConfigsByAPI(r, instance.Spec.Config.Mode, configObj.String()); err != nil {
+			return subResult{err: emperror.Wrap(err, "failed to put emqx config")}
+		}
+
+		instance.Annotations[appsv2beta1.LastEMQXConfigAnnotationKey] = instance.Spec.Config.Data
+		if err := s.Client.Update(ctx, instance); err != nil {
+			return subResult{err: emperror.Wrap(err, "failed to update emqx instance")}
+		}
+		return subResult{}
 	}
 
 	config, err := getEMQXConfigsByAPI(r)