Add ability to scale MariaDB

With this MariaDB can scale to a 3 node Galera cluster.

* Added ProxySQl statefulset
* Updated default version of MariaDB
* Updated Galera helm chart
* Added sensible PDBs to ProxySQL

apis/vshn/v1/dbaas_vshn_mariadb.go

@@ -2,6 +2,7 @@ package v1
 
 import (
 	"fmt"
+
 	xpv1 "github.com/crossplane/crossplane-runtime/apis/common/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 )
@@ -78,12 +79,22 @@ type VSHNMariaDBParameters struct {
 
 	// Security defines the security of a service
 	Security Security `json:"security,omitempty"`
+
+	// +kubebuilder:default=1
+	// +kubebuilder:validation:Enum=1;3;
+
+	// Instances configures the number of MariaDB instances for the cluster.
+	// Each instance contains one MariaDB server.
+	// These serves will form a Galera cluster.
+	// An additional ProxySQL statefulset will be deployed to make failovers
+	// as seamless as possible.
+	Instances int `json:"instances,omitempty"`
 }
 
 // VSHNMariaDBServiceSpec contains MariaDB DBaaS specific properties
 type VSHNMariaDBServiceSpec struct {
-	// +kubebuilder:validation:Enum="10.4";"10.5";"10.6";"10.9";"10.10";"10.11";"11.0";"11.1";"11.2";
-	// +kubebuilder:default="11.2"
+	// +kubebuilder:validation:Enum="10.4";"10.5";"10.6";"10.9";"10.10";"10.11";"11.0";"11.1";"11.2";"11.3";"11.4";"11.5";
+	// +kubebuilder:default="11.5"
 
 	// Version contains supported version of MariaDB.
 	// Multiple versions are supported. The latest version "11.2" is the default version.
@@ -264,7 +275,7 @@ func (v *VSHNMariaDB) GetMonitoring() VSHNMonitoring {
 }
 
 func (v *VSHNMariaDB) GetInstances() int {
-	return 1
+	return v.Spec.Parameters.Instances
 }
 
 func (v *VSHNMariaDB) GetPDBLabels() map[string]string {

crds/vshn.appcat.vshn.io_vshnmariadbs.yaml

@@ -67,6 +67,18 @@ spec:
                           type: string
                       type: object
                       default: {}
+                    instances:
+                      default: 1
+                      description: |-
+                        Instances configures the number of MariaDB instances for the cluster.
+                        Each instance contains one MariaDB server.
+                        These serves will form a Galera cluster.
+                        An additional ProxySQL statefulset will be deployed to make failovers
+                        as seamless as possible.
+                      enum:
+                        - 1
+                        - 3
+                      type: integer
                     maintenance:
                       description: Maintenance contains settings to control the maintenance of an instance.
                       properties:
@@ -4895,7 +4907,7 @@ spec:
                             - guaranteed
                           type: string
                         version:
-                          default: "11.2"
+                          default: "11.5"
                           description: |-
                             Version contains supported version of MariaDB.
                             Multiple versions are supported. The latest version "11.2" is the default version.
@@ -4909,6 +4921,9 @@ spec:
                             - "11.0"
                             - "11.1"
                             - "11.2"
+                            - "11.3"
+                            - "11.4"
+                            - "11.5"
                           type: string
                       type: object
                       default: {}

crds/vshn.appcat.vshn.io_xvshnmariadbs.yaml

@@ -111,6 +111,18 @@ spec:
                           (\*|([1-9]|1[0-2])|\*\/([1-9]|1[0-2])) (\*|([0-6])|\*\/([0-6]))$
                         type: string
                     type: object
+                  instances:
+                    default: 1
+                    description: |-
+                      Instances configures the number of MariaDB instances for the cluster.
+                      Each instance contains one MariaDB server.
+                      These serves will form a Galera cluster.
+                      An additional ProxySQL statefulset will be deployed to make failovers
+                      as seamless as possible.
+                    enum:
+                    - 1
+                    - 3
+                    type: integer
                   maintenance:
                     description: Maintenance contains settings to control the maintenance
                       of an instance.
@@ -5623,7 +5635,7 @@ spec:
                         - guaranteed
                         type: string
                       version:
-                        default: "11.2"
+                        default: "11.5"
                         description: |-
                           Version contains supported version of MariaDB.
                           Multiple versions are supported. The latest version "11.2" is the default version.
@@ -5637,6 +5649,9 @@ spec:
                         - "11.0"
                         - "11.1"
                         - "11.2"
+                        - "11.3"
+                        - "11.4"
+                        - "11.5"
                         type: string
                     type: object
                   size:

pkg/comp-functions/functions/common/tls.go

@@ -13,7 +13,9 @@ import (
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 )
 
-func CreateTlsCerts(ctx context.Context, ns string, serviceName string, svc *runtime.ServiceRuntime) error {
+// CreateTLSCerts creates ssl/tls certificates. Servicename will be concatenated with the given namespace to generate a proper k8s fqdn.
+// In addition to an error it also returns the name of the secret containing the server certifcates.
+func CreateTLSCerts(ctx context.Context, ns string, serviceName string, svc *runtime.ServiceRuntime, additionalSANs ...string) (string, error) {
 
 	selfSignedIssuer := &cmv1.Issuer{
 		ObjectMeta: metav1.ObjectMeta{
@@ -32,9 +34,11 @@ func CreateTlsCerts(ctx context.Context, ns string, serviceName string, svc *run
 	err := svc.SetDesiredKubeObject(selfSignedIssuer, serviceName+"-selfsigned-issuer")
 	if err != nil {
 		err = fmt.Errorf("cannot create selfSignedIssuer object: %w", err)
-		return err
+		return "", err
 	}
 
+	serverCertsSecret := "tls-server-certificate"
+
 	caCert := &cmv1.Certificate{
 
 		ObjectMeta: metav1.ObjectMeta{
@@ -72,7 +76,7 @@ func CreateTlsCerts(ctx context.Context, ns string, serviceName string, svc *run
 	err = svc.SetDesiredKubeObject(caCert, serviceName+"-ca-cert")
 	if err != nil {
 		err = fmt.Errorf("cannot create caCert object: %w", err)
-		return err
+		return serverCertsSecret, err
 	}
 
 	caIssuer := &cmv1.Issuer{
@@ -92,7 +96,7 @@ func CreateTlsCerts(ctx context.Context, ns string, serviceName string, svc *run
 	err = svc.SetDesiredKubeObject(caIssuer, serviceName+"-ca-issuer")
 	if err != nil {
 		err = fmt.Errorf("cannot create caIssuer object: %w", err)
-		return err
+		return serverCertsSecret, err
 	}
 
 	serverCert := &cmv1.Certificate{
@@ -101,7 +105,7 @@ func CreateTlsCerts(ctx context.Context, ns string, serviceName string, svc *run
 			Namespace: ns,
 		},
 		Spec: cmv1.CertificateSpec{
-			SecretName: "tls-server-certificate",
+			SecretName: serverCertsSecret,
 			Duration: &metav1.Duration{
 				Duration: time.Duration(87600 * time.Hour),
 			},
@@ -132,13 +136,15 @@ func CreateTlsCerts(ctx context.Context, ns string, serviceName string, svc *run
 		},
 	}
 
+	serverCert.Spec.DNSNames = append(serverCert.Spec.DNSNames, additionalSANs...)
+
 	cd := []xkube.ConnectionDetail{
 		{
 			ObjectReference: corev1.ObjectReference{
 				APIVersion: "v1",
 				Kind:       "Secret",
 				Namespace:  ns,
-				Name:       "tls-server-certificate",
+				Name:       serverCertsSecret,
 				FieldPath:  "data[ca.crt]",
 			},
 			ToConnectionSecretKey: "ca.crt",
@@ -148,7 +154,7 @@ func CreateTlsCerts(ctx context.Context, ns string, serviceName string, svc *run
 				APIVersion: "v1",
 				Kind:       "Secret",
 				Namespace:  ns,
-				Name:       "tls-server-certificate",
+				Name:       serverCertsSecret,
 				FieldPath:  "data[tls.crt]",
 			},
 			ToConnectionSecretKey: "tls.crt",
@@ -158,7 +164,7 @@ func CreateTlsCerts(ctx context.Context, ns string, serviceName string, svc *run
 				APIVersion: "v1",
 				Kind:       "Secret",
 				Namespace:  ns,
-				Name:       "tls-server-certificate",
+				Name:       serverCertsSecret,
 				FieldPath:  "data[tls.key]",
 			},
 			ToConnectionSecretKey: "tls.key",
@@ -168,9 +174,9 @@ func CreateTlsCerts(ctx context.Context, ns string, serviceName string, svc *run
 	err = svc.SetDesiredKubeObject(serverCert, serviceName+"-server-cert", runtime.KubeOptionAddConnectionDetails(ns, cd...))
 	if err != nil {
 		err = fmt.Errorf("cannot create serverCert object: %w", err)
-		return err
+		return serverCertsSecret, err
 	}
 
-	return nil
+	return serverCertsSecret, nil
 
 }

pkg/comp-functions/functions/vshnkeycloak/deploy.go

@@ -122,7 +122,7 @@ func DeployKeycloak(ctx context.Context, comp *vshnv1.VSHNKeycloak, svc *runtime
 
 	svc.Log.Info("Creating Keycloak TLS certs")
 	// The helm chart appends `-keycloakx-http` to the http service.
-	err = common.CreateTlsCerts(ctx, comp.GetInstanceNamespace(), comp.GetName()+"-keycloakx-http", svc)
+	_, err = common.CreateTLSCerts(ctx, comp.GetInstanceNamespace(), comp.GetName()+"-keycloakx-http", svc)
 	if err != nil {
 		return runtime.NewWarningResult(fmt.Sprintf("cannot add tls certificate: %s", err))
 	}

pkg/comp-functions/functions/vshnmariadb/mariadb_deploy.go

@@ -21,6 +21,7 @@ import (
 	corev1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
+	"k8s.io/apimachinery/pkg/util/intstr"
 	"k8s.io/utils/ptr"
 )
 
@@ -53,8 +54,27 @@ func DeployMariadb(ctx context.Context, comp *vshnv1.VSHNMariaDB, svc *runtime.S
 		return runtime.NewWarningResult(fmt.Errorf("cannot bootstrap instance namespace: %w", err).Error())
 	}
 
+	svc.Log.Info("Creating main mariadb service")
+	err = createMainService(comp, svc, passwordSecret)
+	if err != nil {
+		return runtime.NewWarningResult(fmt.Sprintf("cannot create service: %s", err))
+	}
+
+	// To make scaling up and down as seamless as possible we create a cert that also includes the
+	// proxy dns names.
 	l.Info("Creating tls certificate for mariadb instance")
-	err = common.CreateTlsCerts(ctx, comp.GetInstanceNamespace(), comp.GetName(), svc)
+	_, err = common.CreateTLSCerts(ctx, comp.GetInstanceNamespace(), comp.GetName(), svc,
+		comp.GetName(),
+		fmt.Sprintf("*.%s-headless.%s.svc.cluster.local", comp.GetName(), comp.GetInstanceNamespace()),
+		fmt.Sprintf("*.%s.%s.svc.cluster.local", comp.GetName(), comp.GetInstanceNamespace()),
+		fmt.Sprintf("%s-headless.%s.svc.cluster.local", comp.GetName(), comp.GetInstanceNamespace()),
+		fmt.Sprintf("%s.%s.svc.cluster.local", comp.GetName(), comp.GetInstanceNamespace()),
+		fmt.Sprintf("mariadb.%s.svc.cluster.local", comp.GetInstanceNamespace()),
+		fmt.Sprintf("mariadb.%s.svc", comp.GetInstanceNamespace()),
+		fmt.Sprintf("proxysql-0.proxysqlcluster.%s.svc", comp.GetInstanceNamespace()),
+		fmt.Sprintf("proxysql-1.proxysqlcluster.%s.svc", comp.GetInstanceNamespace()),
+	)
+
 	if err != nil {
 		return runtime.NewWarningResult(fmt.Errorf("cannot create tls certificate: %w", err).Error())
 	}
@@ -112,25 +132,14 @@ func createObjectHelmRelease(ctx context.Context, comp *vshnv1.VSHNMariaDB, svc
 }
 
 func getConnectionDetails(comp *vshnv1.VSHNMariaDB, svc *runtime.ServiceRuntime, secretName string) error {
-	secret := &corev1.Secret{}
-
-	err := svc.GetObservedKubeObject(secret, secretName)
 
+	mariadbRootPw, err := getMariaDBRootPassword(secretName, svc)
 	if err != nil {
-		if err == runtime.ErrNotFound {
-			return nil
-		}
 		return err
 	}
-	mariadbRootPw := secret.Data["mariadb-root-password"]
 
-	mariadbHost := comp.GetName() + ".vshn-mariadb-" + comp.GetName() + ".svc.cluster.local"
-	mariadbURL := fmt.Sprintf("mysql://%s:%s@%s:%s", mariadbUser, mariadbRootPw, mariadbHost, mariadbPort)
-
-	svc.SetConnectionDetail("MARIADB_HOST", []byte(mariadbHost))
 	svc.SetConnectionDetail("MARIADB_PORT", []byte(mariadbPort))
 	svc.SetConnectionDetail("MARIADB_USERNAME", []byte(mariadbUser))
-	svc.SetConnectionDetail("MARIADB_URL", []byte(mariadbURL))
 	svc.SetConnectionDetail("MARIADB_PASSWORD", mariadbRootPw)
 
 	return nil
@@ -161,7 +170,7 @@ func newValues(ctx context.Context, svc *runtime.ServiceRuntime, comp *vshnv1.VS
 	values = map[string]interface{}{
 		"existingSecret":   secretName,
 		"fullnameOverride": comp.GetName(),
-		"replicaCount":     1,
+		"replicaCount":     comp.GetInstances(),
 		"resources": map[string]interface{}{
 			"requests": map[string]interface{}{
 				"memory": res.ReqMem.String(),
@@ -186,8 +195,11 @@ func newValues(ctx context.Context, svc *runtime.ServiceRuntime, comp *vshnv1.VS
 			"size":         res.Disk.String(),
 			"storageClass": comp.Spec.Parameters.StorageClass,
 		},
+		// We don't need the startup probe for Galera clusters, as ProxySQL
+		// will check the state independetly and is usually faster than the probe.
+		// Also for single instances it unnecessarily slows downt he provisioning.
 		"startupProbe": map[string]interface{}{
-			"enabled": true,
+			"enabled": false,
 		},
 		"metrics": map[string]interface{}{
 			"enabled": true,
@@ -208,6 +220,9 @@ func newValues(ctx context.Context, svc *runtime.ServiceRuntime, comp *vshnv1.VS
 			"enabled": !svc.GetBoolFromCompositionConfig("isOpenshift"),
 		},
 		"nodeSelector": nodeSelector,
+		"podLabels": map[string]string{
+			"app": "mariadb",
+		},
 	}
 
 	return values, nil
@@ -396,3 +411,64 @@ func getReleaseSecretValue(svc *runtime.ServiceRuntime, comp *vshnv1.VSHNMariaDB
 	}
 	return m["release"]
 }
+
+// To make scaling as seamless as possible we create an additional service.
+// This service will either point to a single db instance, or to the proxysql cluster.
+func createMainService(comp *vshnv1.VSHNMariaDB, svc *runtime.ServiceRuntime, secretName string) error {
+	target := map[string]string{}
+	targetPort := 3306
+	serviceName := "mariadb"
+
+	if comp.GetInstances() == 1 {
+		target["app"] = "mariadb"
+	} else {
+		target["app"] = "proxysql"
+		targetPort = 6033
+	}
+
+	service := corev1.Service{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      serviceName,
+			Namespace: comp.GetInstanceNamespace(),
+		},
+		Spec: corev1.ServiceSpec{
+			Selector: target,
+			Type:     corev1.ServiceTypeClusterIP,
+			Ports: []corev1.ServicePort{
+				{
+					Name:       serviceName,
+					Protocol:   corev1.ProtocolTCP,
+					TargetPort: intstr.FromInt(targetPort),
+					Port:       3306,
+				},
+			},
+		},
+	}
+
+	mariadbRootPw, err := getMariaDBRootPassword(secretName, svc)
+	if err != nil {
+		return err
+	}
+
+	mariadbHost := serviceName + ".vshn-mariadb-" + comp.GetName() + ".svc.cluster.local"
+	mariadbURL := fmt.Sprintf("mysql://%s:%s@%s:%s", mariadbUser, mariadbRootPw, mariadbHost, mariadbPort)
+
+	svc.SetConnectionDetail("MARIADB_HOST", []byte(mariadbHost))
+	svc.SetConnectionDetail("MARIADB_URL", []byte(mariadbURL))
+
+	return svc.SetDesiredKubeObject(&service, comp.GetName()+"-main-service")
+}
+
+func getMariaDBRootPassword(secretName string, svc *runtime.ServiceRuntime) ([]byte, error) {
+	secret := &corev1.Secret{}
+
+	err := svc.GetObservedKubeObject(secret, secretName)
+
+	if err != nil {
+		if err == runtime.ErrNotFound {
+			return nil, nil
+		}
+		return nil, err
+	}
+	return secret.Data["mariadb-root-password"], nil
+}

pkg/comp-functions/functions/vshnmariadb/mariadb_deploy_test.go

@@ -22,9 +22,9 @@ func TestMariadbDeploy(t *testing.T) {
 
 	rootUser := "root"
 	rootPassword := "mariadb123"
-	mariadbHost := "mariadb-gc9x4.vshn-mariadb-mariadb-gc9x4.svc.cluster.local"
+	mariadbHost := "mariadb.vshn-mariadb-mariadb-gc9x4.svc.cluster.local"
 	mariadbPort := "3306"
-	mariadbUrl := "mysql://root:mariadb123@mariadb-gc9x4.vshn-mariadb-mariadb-gc9x4.svc.cluster.local:3306"
+	mariadbUrl := "mysql://root:[email protected]:3306"
 
 	assert.Nil(t, DeployMariadb(ctx, &vshnv1.VSHNMariaDB{}, svc))