[occm] add max_retries_down support for octavia health monitors (#2372

) (#2380)

docs/openstack-cloud-controller-manager/expose-applications-using-loadbalancer-type-service.md

@@ -192,7 +192,11 @@ Request Body:
 
 - `loadbalancer.openstack.org/health-monitor-max-retries`
 
-  Defines the health monitor retry count for the loadbalancer pools.
+  Defines the health monitor retry count for the loadbalancer pool members.
+
+- `loadbalancer.openstack.org/health-monitor-max-retries-down`
+
+  Defines the health monitor retry count for the loadbalancer pool members to be marked down.
 
 - `loadbalancer.openstack.org/flavor-id`
 
@@ -255,7 +259,8 @@ subnet-id="fa6a4e6c-6ae4-4dde-ae86-3e2f452c1f03"
 create-monitor=true
 monitor-delay=60s
 monitor-timeout=30s
-monitor-max-retries=5
+monitor-max-retries=1
+monitor-max-retries-down=3
 
 [LoadBalancerClass "internetFacing"]
 floating-network-id="c57af0a0-da92-49be-a98a-345ceca004b3"

docs/openstack-cloud-controller-manager/using-openstack-cloud-controller-manager.md

@@ -229,6 +229,9 @@ Although the openstack-cloud-controller-manager was initially implemented with N
 * `monitor-max-retries`
   The number of successful checks before changing the operating status of the load balancer member to ONLINE. A valid value is from 1 to 10. Default: 1
 
+* `monitor-max-retries-down`
+  The number of unsuccessful checks before changing the operating status of the load balancer member to ERROR. A valid value is from 1 to 10. Default: 3
+
 * `monitor-timeout`
   The maximum time, in seconds, that a monitor waits to connect backend before it times out. Default: 3
 

pkg/openstack/loadbalancer.go

@@ -88,12 +88,13 @@ const (
 	ServiceAnnotationLoadBalancerAvailabilityZone     = "loadbalancer.openstack.org/availability-zone"
 	// ServiceAnnotationLoadBalancerEnableHealthMonitor defines whether to create health monitor for the load balancer
 	// pool, if not specified, use 'create-monitor' config. The health monitor can be created or deleted dynamically.
-	ServiceAnnotationLoadBalancerEnableHealthMonitor     = "loadbalancer.openstack.org/enable-health-monitor"
-	ServiceAnnotationLoadBalancerHealthMonitorDelay      = "loadbalancer.openstack.org/health-monitor-delay"
-	ServiceAnnotationLoadBalancerHealthMonitorTimeout    = "loadbalancer.openstack.org/health-monitor-timeout"
-	ServiceAnnotationLoadBalancerHealthMonitorMaxRetries = "loadbalancer.openstack.org/health-monitor-max-retries"
-	ServiceAnnotationLoadBalancerLoadbalancerHostname    = "loadbalancer.openstack.org/hostname"
-	ServiceAnnotationLoadBalancerAddress                 = "loadbalancer.openstack.org/load-balancer-address"
+	ServiceAnnotationLoadBalancerEnableHealthMonitor         = "loadbalancer.openstack.org/enable-health-monitor"
+	ServiceAnnotationLoadBalancerHealthMonitorDelay          = "loadbalancer.openstack.org/health-monitor-delay"
+	ServiceAnnotationLoadBalancerHealthMonitorTimeout        = "loadbalancer.openstack.org/health-monitor-timeout"
+	ServiceAnnotationLoadBalancerHealthMonitorMaxRetries     = "loadbalancer.openstack.org/health-monitor-max-retries"
+	ServiceAnnotationLoadBalancerHealthMonitorMaxRetriesDown = "loadbalancer.openstack.org/health-monitor-max-retries-down"
+	ServiceAnnotationLoadBalancerLoadbalancerHostname        = "loadbalancer.openstack.org/hostname"
+	ServiceAnnotationLoadBalancerAddress                     = "loadbalancer.openstack.org/load-balancer-address"
 	// revive:disable:var-naming
 	ServiceAnnotationTlsContainerRef = "loadbalancer.openstack.org/default-tls-container-ref"
 	// revive:enable:var-naming
@@ -325,33 +326,34 @@ func tagList(tags string) ([]string, bool, bool) {
 
 // serviceConfig contains configurations for creating a Service.
 type serviceConfig struct {
-	internal                bool
-	connLimit               int
-	configClassName         string
-	lbNetworkID             string
-	lbSubnetID              string
-	lbMemberSubnetID        string
-	lbPublicNetworkID       string
-	lbPublicSubnetSpec      *floatingSubnetSpec
-	keepClientIP            bool
-	enableProxyProtocol     bool
-	timeoutClientData       int
-	timeoutMemberConnect    int
-	timeoutMemberData       int
-	timeoutTCPInspect       int
-	allowedCIDR             []string
-	enableMonitor           bool
-	flavorID                string
-	availabilityZone        string
-	tlsContainerRef         string
-	lbID                    string
-	lbName                  string
-	supportLBTags           bool
-	healthCheckNodePort     int
-	healthMonitorDelay      int
-	healthMonitorTimeout    int
-	healthMonitorMaxRetries int
-	preferredIPFamily       corev1.IPFamily // preferred (the first) IP family indicated in service's `spec.ipFamilies`
+	internal                    bool
+	connLimit                   int
+	configClassName             string
+	lbNetworkID                 string
+	lbSubnetID                  string
+	lbMemberSubnetID            string
+	lbPublicNetworkID           string
+	lbPublicSubnetSpec          *floatingSubnetSpec
+	keepClientIP                bool
+	enableProxyProtocol         bool
+	timeoutClientData           int
+	timeoutMemberConnect        int
+	timeoutMemberData           int
+	timeoutTCPInspect           int
+	allowedCIDR                 []string
+	enableMonitor               bool
+	flavorID                    string
+	availabilityZone            string
+	tlsContainerRef             string
+	lbID                        string
+	lbName                      string
+	supportLBTags               bool
+	healthCheckNodePort         int
+	healthMonitorDelay          int
+	healthMonitorTimeout        int
+	healthMonitorMaxRetries     int
+	healthMonitorMaxRetriesDown int
+	preferredIPFamily           corev1.IPFamily // preferred (the first) IP family indicated in service's `spec.ipFamilies`
 }
 
 type listenerKey struct {
@@ -1091,11 +1093,15 @@ func (lbaas *LbaasV2) ensureOctaviaHealthMonitor(lbID string, name string, pool
 			}
 			monitorID = ""
 		}
-		if svcConf.healthMonitorDelay != monitor.Delay || svcConf.healthMonitorTimeout != monitor.Timeout || svcConf.healthMonitorMaxRetries != monitor.MaxRetries {
+		if svcConf.healthMonitorDelay != monitor.Delay ||
+			svcConf.healthMonitorTimeout != monitor.Timeout ||
+			svcConf.healthMonitorMaxRetries != monitor.MaxRetries ||
+			svcConf.healthMonitorMaxRetriesDown != monitor.MaxRetriesDown {
 			updateOpts := v2monitors.UpdateOpts{
-				Delay:      svcConf.healthMonitorDelay,
-				Timeout:    svcConf.healthMonitorTimeout,
-				MaxRetries: svcConf.healthMonitorMaxRetries,
+				Delay:          svcConf.healthMonitorDelay,
+				Timeout:        svcConf.healthMonitorTimeout,
+				MaxRetries:     svcConf.healthMonitorMaxRetries,
+				MaxRetriesDown: svcConf.healthMonitorMaxRetriesDown,
 			}
 			klog.Infof("Updating health monitor %s updateOpts %+v", monitorID, updateOpts)
 			if err := openstackutil.UpdateHealthMonitor(lbaas.lb, monitorID, updateOpts); err != nil {
@@ -1142,10 +1148,11 @@ func (lbaas *LbaasV2) canUseHTTPMonitor(port corev1.ServicePort) bool {
 // buildMonitorCreateOpts returns a v2monitors.CreateOpts without PoolID for consumption of both, fully popuplated Loadbalancers and Monitors.
 func (lbaas *LbaasV2) buildMonitorCreateOpts(svcConf *serviceConfig, port corev1.ServicePort) v2monitors.CreateOpts {
 	opts := v2monitors.CreateOpts{
-		Type:       string(port.Protocol),
-		Delay:      svcConf.healthMonitorDelay,
-		Timeout:    svcConf.healthMonitorTimeout,
-		MaxRetries: svcConf.healthMonitorMaxRetries,
+		Type:           string(port.Protocol),
+		Delay:          svcConf.healthMonitorDelay,
+		Timeout:        svcConf.healthMonitorTimeout,
+		MaxRetries:     svcConf.healthMonitorMaxRetries,
+		MaxRetriesDown: svcConf.healthMonitorMaxRetriesDown,
 	}
 	if port.Protocol == corev1.ProtocolUDP {
 		opts.Type = "UDP-CONNECT"
@@ -1580,6 +1587,7 @@ func (lbaas *LbaasV2) checkServiceUpdate(service *corev1.Service, nodes []*corev
 	svcConf.healthMonitorDelay = getIntFromServiceAnnotation(service, ServiceAnnotationLoadBalancerHealthMonitorDelay, int(lbaas.opts.MonitorDelay.Duration.Seconds()))
 	svcConf.healthMonitorTimeout = getIntFromServiceAnnotation(service, ServiceAnnotationLoadBalancerHealthMonitorTimeout, int(lbaas.opts.MonitorTimeout.Duration.Seconds()))
 	svcConf.healthMonitorMaxRetries = getIntFromServiceAnnotation(service, ServiceAnnotationLoadBalancerHealthMonitorMaxRetries, int(lbaas.opts.MonitorMaxRetries))
+	svcConf.healthMonitorMaxRetriesDown = getIntFromServiceAnnotation(service, ServiceAnnotationLoadBalancerHealthMonitorMaxRetriesDown, int(lbaas.opts.MonitorMaxRetriesDown))
 	return nil
 }
 
@@ -1815,6 +1823,7 @@ func (lbaas *LbaasV2) checkService(service *corev1.Service, nodes []*corev1.Node
 	svcConf.healthMonitorDelay = getIntFromServiceAnnotation(service, ServiceAnnotationLoadBalancerHealthMonitorDelay, int(lbaas.opts.MonitorDelay.Duration.Seconds()))
 	svcConf.healthMonitorTimeout = getIntFromServiceAnnotation(service, ServiceAnnotationLoadBalancerHealthMonitorTimeout, int(lbaas.opts.MonitorTimeout.Duration.Seconds()))
 	svcConf.healthMonitorMaxRetries = getIntFromServiceAnnotation(service, ServiceAnnotationLoadBalancerHealthMonitorMaxRetries, int(lbaas.opts.MonitorMaxRetries))
+	svcConf.healthMonitorMaxRetriesDown = getIntFromServiceAnnotation(service, ServiceAnnotationLoadBalancerHealthMonitorMaxRetriesDown, int(lbaas.opts.MonitorMaxRetriesDown))
 	return nil
 }
 

pkg/openstack/openstack.go

@@ -95,6 +95,7 @@ type LoadBalancerOpts struct {
 	MonitorDelay          util.MyDuration     `gcfg:"monitor-delay"`
 	MonitorTimeout        util.MyDuration     `gcfg:"monitor-timeout"`
 	MonitorMaxRetries     uint                `gcfg:"monitor-max-retries"`
+	MonitorMaxRetriesDown uint                `gcfg:"monitor-max-retries-down"`
 	ManageSecurityGroups  bool                `gcfg:"manage-security-groups"`
 	NodeSecurityGroupIDs  []string            // Do not specify, get it automatically when enable manage-security-groups. TODO(FengyunPan): move it into cache
 	InternalLB            bool                `gcfg:"internal-lb"` // default false
@@ -205,6 +206,7 @@ func ReadConfig(config io.Reader) (Config, error) {
 	cfg.LoadBalancer.MonitorDelay = util.MyDuration{Duration: 5 * time.Second}
 	cfg.LoadBalancer.MonitorTimeout = util.MyDuration{Duration: 3 * time.Second}
 	cfg.LoadBalancer.MonitorMaxRetries = 1
+	cfg.LoadBalancer.MonitorMaxRetriesDown = 3
 	cfg.LoadBalancer.CascadeDelete = true
 	cfg.LoadBalancer.EnableIngressHostname = false
 	cfg.LoadBalancer.IngressHostnameSuffix = defaultProxyHostnameSuffix

pkg/openstack/openstack_test.go

@@ -98,7 +98,8 @@ func TestReadConfig(t *testing.T) {
  create-monitor = yes
  monitor-delay = 1m
  monitor-timeout = 30s
- monitor-max-retries = 3
+ monitor-max-retries = 1
+ monitor-max-retries-down = 3
  [Metadata]
  search-order = configDrive, metadataService
  `))
@@ -131,16 +132,19 @@ func TestReadConfig(t *testing.T) {
 	}
 
 	if !cfg.LoadBalancer.CreateMonitor {
-		t.Errorf("incorrect lb.createmonitor: %t", cfg.LoadBalancer.CreateMonitor)
+		t.Errorf("incorrect lb.create-monitor: %t", cfg.LoadBalancer.CreateMonitor)
 	}
 	if cfg.LoadBalancer.MonitorDelay.Duration != 1*time.Minute {
-		t.Errorf("incorrect lb.monitordelay: %s", cfg.LoadBalancer.MonitorDelay)
+		t.Errorf("incorrect lb.monitor-delay: %s", cfg.LoadBalancer.MonitorDelay)
 	}
 	if cfg.LoadBalancer.MonitorTimeout.Duration != 30*time.Second {
-		t.Errorf("incorrect lb.monitortimeout: %s", cfg.LoadBalancer.MonitorTimeout)
+		t.Errorf("incorrect lb.monitor-timeout: %s", cfg.LoadBalancer.MonitorTimeout)
 	}
-	if cfg.LoadBalancer.MonitorMaxRetries != 3 {
-		t.Errorf("incorrect lb.monitormaxretries: %d", cfg.LoadBalancer.MonitorMaxRetries)
+	if cfg.LoadBalancer.MonitorMaxRetries != 1 {
+		t.Errorf("incorrect lb.monitor-max-retries: %d", cfg.LoadBalancer.MonitorMaxRetries)
+	}
+	if cfg.LoadBalancer.MonitorMaxRetriesDown != 3 {
+		t.Errorf("incorrect lb.monitor-max-retries-down: %d", cfg.LoadBalancer.MonitorMaxRetriesDown)
 	}
 	if cfg.Metadata.SearchOrder != "configDrive, metadataService" {
 		t.Errorf("incorrect md.search-order: %v", cfg.Metadata.SearchOrder)
@@ -187,7 +191,8 @@ clouds:
  create-monitor = yes
  monitor-delay = 1m
  monitor-timeout = 30s
- monitor-max-retries = 3
+ monitor-max-retries = 1
+ monitor-max-retries-down = 3
  [Metadata]
  search-order = configDrive, metadataService
 `))
@@ -227,7 +232,15 @@ clouds:
 
 	// Make non-global sections dont get overwritten
 	if !cfg.LoadBalancer.CreateMonitor {
-		t.Errorf("incorrect lb.createmonitor: %t", cfg.LoadBalancer.CreateMonitor)
+		t.Errorf("incorrect lb.create-monitor: %t", cfg.LoadBalancer.CreateMonitor)
+	}
+
+	if cfg.LoadBalancer.MonitorMaxRetries != 1 {
+		t.Errorf("incorrect lb.monitor-max-retries: %d", cfg.LoadBalancer.MonitorMaxRetries)
+	}
+
+	if cfg.LoadBalancer.MonitorMaxRetriesDown != 3 {
+		t.Errorf("incorrect lb.monitor-max-retries-down: %d", cfg.LoadBalancer.MonitorMaxRetriesDown)
 	}
 }