OVN DB TLS support

Accepts a TLS secret name containing the OVN DB client certs/key and CA cert.

Jira: OSPRH-2191

Depends-On: openstack-k8s-operators/ovn-operator#234

api/bases/neutron.openstack.org_neutronapis.yaml

@@ -2250,6 +2250,14 @@ spec:
                     description: CaBundleSecretName - holding the CA certs in a pre-created
                       bundle file
                     type: string
+                  ovndb:
+                    description: OvnDb GenericService - holds the secret for the OvnDb
+                      client cert
+                    properties:
+                      secretName:
+                        description: SecretName - holding the cert, key for the service
+                        type: string
+                    type: object
                 type: object
             required:
             - containerImage

api/go.mod

@@ -71,3 +71,5 @@ require (
 // mschuppert: map to latest commit from release-4.13 tag
 // must consistent within modules and service operators
 replace github.com/openshift/api => github.com/openshift/api v0.0.0-20230414143018-3367bc7e6ac7 //allow-merging
+
+replace github.com/openstack-k8s-operators/ovn-operator/api => github.com/olliewalsh/ovn-operator/api v0.0.0-20240227125719-3f68307e9041

api/v1beta1/neutronapi_types.go

@@ -131,7 +131,22 @@ type NeutronAPISpec struct {
 	// +kubebuilder:validation:Optional
 	// +operator-sdk:csv:customresourcedefinitions:type=spec
 	// TLS - Parameters related to the TLS
-	TLS tls.API `json:"tls,omitempty"`
+	TLS NeutronApiTLS `json:"tls,omitempty"`
+}
+
+type NeutronApiTLS struct {
+	// +kubebuilder:validation:optional
+	// +operator-sdk:csv:customresourcedefinitions:type=spec
+	// API tls type which encapsulates for API services
+	API tls.APIService `json:"api,omitempty"`
+	// +kubebuilder:validation:optional
+	// +operator-sdk:csv:customresourcedefinitions:type=spec
+	// Secret containing CA bundle
+	tls.Ca `json:",inline"`
+	// +kubebuilder:validation:optional
+	// +operator-sdk:csv:customresourcedefinitions:type=spec
+	// OvnDb GenericService - holds the secret for the OvnDb client cert
+	OvnDb tls.GenericService `json:"ovndb,omitempty"`
 }
 
 // APIOverrideSpec to override the generated manifest of several child resources.

config/crd/bases/neutron.openstack.org_neutronapis.yaml

@@ -2250,6 +2250,14 @@ spec:
                     description: CaBundleSecretName - holding the CA certs in a pre-created
                       bundle file
                     type: string
+                  ovndb:
+                    description: OvnDb GenericService - holds the secret for the OvnDb
+                      client cert
+                    properties:
+                      secretName:
+                        description: SecretName - holding the cert, key for the service
+                        type: string
+                    type: object
                 type: object
             required:
             - containerImage

controllers/neutronapi_controller.go

@@ -212,13 +212,15 @@ const (
 	caBundleSecretNameField = ".spec.tls.caBundleSecretName"
 	tlsAPIInternalField     = ".spec.tls.api.internal.secretName"
 	tlsAPIPublicField       = ".spec.tls.api.public.secretName"
+	tlsAPIOvnDbField        = ".spec.tls.api.ovndb.secretName"
 )
 
 var allWatchFields = []string{
 	passwordSecretField,
 	caBundleSecretNameField,
 	tlsAPIInternalField,
 	tlsAPIPublicField,
+	tlsAPIOvnDbField,
 }
 
 // SetupWithManager -
@@ -271,6 +273,18 @@ func (r *NeutronAPIReconciler) SetupWithManager(ctx context.Context, mgr ctrl.Ma
 		return err
 	}
 
+	// index tlsAPIOvnDbField
+	if err := mgr.GetFieldIndexer().IndexField(context.Background(), &neutronv1beta1.NeutronAPI{}, tlsAPIOvnDbField, func(rawObj client.Object) []string {
+		// Extract the secret name from the spec, if one is provided
+		cr := rawObj.(*neutronv1beta1.NeutronAPI)
+		if cr.Spec.TLS.OvnDb.SecretName == nil {
+			return nil
+		}
+		return []string{*cr.Spec.TLS.OvnDb.SecretName}
+	}); err != nil {
+		return err
+	}
+
 	crs := &neutronv1beta1.NeutronAPIList{}
 	return ctrl.NewControllerManagedBy(mgr).
 		For(&neutronv1beta1.NeutronAPI{}).
@@ -1284,6 +1298,7 @@ func (r *NeutronAPIReconciler) ensureExternalMetadataAgentSecret(
 	}
 	templateParameters := make(map[string]interface{})
 	templateParameters["SBConnection"] = sbEndpoint
+	templateParameters["OVNDB_TLS"] = instance.Spec.TLS.OvnDb.Enabled()
 
 	secretName := getMetadataAgentSecretName(instance)
 	return r.ensureExternalSecret(ctx, h, instance, secretName, templates, templateParameters, envVars)
@@ -1303,6 +1318,7 @@ func (r *NeutronAPIReconciler) ensureExternalOvnAgentSecret(
 	templateParameters := make(map[string]interface{})
 	templateParameters["NBConnection"] = nbEndpoint
 	templateParameters["SBConnection"] = sbEndpoint
+	templateParameters["OVNDB_TLS"] = instance.Spec.TLS.OvnDb.Enabled()
 
 	secretName := getOvnAgentSecretName(instance)
 	return r.ensureExternalSecret(ctx, h, instance, secretName, templates, templateParameters, envVars)
@@ -1436,6 +1452,7 @@ func (r *NeutronAPIReconciler) generateServiceSecrets(
 	// OVN
 	templateParameters["NBConnection"] = nbEndpoint
 	templateParameters["SBConnection"] = sbEndpoint
+	templateParameters["OVNDB_TLS"] = instance.Spec.TLS.OvnDb.Enabled()
 
 	// create httpd  vhost template parameters
 	httpdVhostConfig := map[string]interface{}{}

go.mod

@@ -89,3 +89,5 @@ replace github.com/openstack-k8s-operators/neutron-operator/api => ./api
 // mschuppert: map to latest commit from release-4.13 tag
 // must consistent within modules and service operators
 replace github.com/openshift/api => github.com/openshift/api v0.0.0-20230414143018-3367bc7e6ac7 //allow-merging
+
+replace github.com/openstack-k8s-operators/ovn-operator/api => github.com/olliewalsh/ovn-operator/api v0.0.0-20240227125719-3f68307e9041

go.sum

@@ -72,6 +72,8 @@ github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9G
 github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk=
 github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA=
 github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ=
+github.com/olliewalsh/ovn-operator/api v0.0.0-20240227125719-3f68307e9041 h1:AmR1WRKMz3UMfq2g09xA/7lhq43qmdELz7nIUp8l+Io=
+github.com/olliewalsh/ovn-operator/api v0.0.0-20240227125719-3f68307e9041/go.mod h1:1u75r7K/3O7c7WDEn4ItZiZPu8smETxLdRhpi6yt+7I=
 github.com/onsi/ginkgo/v2 v2.15.0 h1:79HwNRBAZHOEwrczrgSOPy+eFTTlIGELKy5as+ClttY=
 github.com/onsi/ginkgo/v2 v2.15.0/go.mod h1:HlxMHtYF57y6Dpf+mc5529KKmSq9h2FpCF+/ZkwUxKM=
 github.com/onsi/gomega v1.31.1 h1:KYppCUK+bUgAZwHOu7EXVBKyQA6ILvOESHkn/tgoqvo=
@@ -92,8 +94,6 @@ github.com/openstack-k8s-operators/lib-common/modules/test v0.3.1-0.202402241824
 github.com/openstack-k8s-operators/lib-common/modules/test v0.3.1-0.20240224182407-3b6c02b195f6/go.mod h1:82nzS+DbBe1tzaMvNHH8FctmZzQ14ZAJysFGsMJiivo=
 github.com/openstack-k8s-operators/mariadb-operator/api v0.3.1-0.20240222094307-76fef735f093 h1:gmm2o5bVYIeuAVHp7WsDIpQc8vh+/9tUUYY4Wfyus/o=
 github.com/openstack-k8s-operators/mariadb-operator/api v0.3.1-0.20240222094307-76fef735f093/go.mod h1:f9IIyWeoskWoeWaDFF3qmAJ2Kqyovfi0Ar/QUfk3qag=
-github.com/openstack-k8s-operators/ovn-operator/api v0.3.1-0.20240221131248-e97a8e5ca98f h1:iacJjeV8yVUE0ZD27PjrbLTgQlaAUF5s+fPczQg/Yqc=
-github.com/openstack-k8s-operators/ovn-operator/api v0.3.1-0.20240221131248-e97a8e5ca98f/go.mod h1:m/5jovuZ3Y1/Uy2af8RqxWhe3+bWn7QIFXH4amKBdmY=
 github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
 github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=

pkg/neutronapi/deployment.go

@@ -112,6 +112,15 @@ func Deployment(
 		}
 	}
 
+	if instance.Spec.TLS.OvnDb.Enabled() {
+		svc := tls.Service{
+			SecretName: *instance.Spec.TLS.OvnDb.SecretName,
+			CaMount:    ptr.To("/var/lib/config-data/tls/certs/ovndbca.crt"),
+		}
+		volumes = append(volumes, svc.CreateVolume("ovndb"))
+		apiVolumeMounts = append(apiVolumeMounts, svc.CreateVolumeMounts("ovndb")...)
+	}
+
 	deployment := &appsv1.Deployment{
 		ObjectMeta: metav1.ObjectMeta{
 			Name:      ServiceName,

pkg/neutronapi/volumes.go

@@ -43,7 +43,7 @@ func GetVolumeMounts(serviceName string, extraVol []neutronv1beta1.NeutronExtraV
 	res := []corev1.VolumeMount{
 		{
 			Name:      "config",
-			MountPath: "/var/lib/config-data",
+			MountPath: "/var/lib/config-data/default",
 			ReadOnly:  true,
 		},
 		{

templates/neutronapi/config/01-neutron.conf

@@ -44,6 +44,14 @@ ovn_sb_connection = {{ .SBConnection }}
 ovn_l3_scheduler = leastloaded
 ovn_metadata_enabled = True
 enable_distributed_floating_ip=True
+{{- if .OVNDB_TLS }}
+ovn_nb_private_key = /etc/pki/tls/private/ovndb.key
+ovn_nb_certificate = /etc/pki/tls/certs/ovndb.crt
+ovn_nb_ca_cert = /etc/pki/tls/certs/ovndbca.crt
+ovn_sb_private_key = /etc/pki/tls/private/ovndb.key
+ovn_sb_certificate = /etc/pki/tls/certs/ovndb.crt
+ovn_sb_ca_cert = /etc/pki/tls/certs/ovndbca.crt
+{{- end }}
 
 [keystone_authtoken]
 www_authenticate_uri = {{ .KeystonePublicURL }}

templates/neutronapi/config/db-sync-config.json

@@ -2,19 +2,19 @@
   "command": "neutron-db-manage --config-file /usr/share/neutron/neutron-dist.conf --config-file /etc/neutron/neutron.conf --config-dir /etc/neutron/neutron.conf.d upgrade heads",
   "config_files": [
     {
-      "source": "/var/lib/config-data/01-neutron.conf",
+      "source": "/var/lib/config-data/default/01-neutron.conf",
       "dest": "/etc/neutron/neutron.conf.d/01-neutron.conf",
       "owner": "root:neutron",
       "perm": "0640"
     },
     {
-      "source": "/var/lib/config-data/02-neutron-custom.conf",
+      "source": "/var/lib/config-data/default/02-neutron-custom.conf",
       "dest": "/etc/neutron/neutron.conf.d/02-neutron-custom.conf",
       "owner": "root:neutron",
       "perm": "0640"
     },
     {
-      "source": "/var/lib/config-data/my.cnf",
+      "source": "/var/lib/config-data/default/my.cnf",
       "dest": "/etc/my.cnf",
       "owner": "neutron",
       "perm": "0644"

templates/neutronapi/config/neutron-api-config.json

@@ -2,22 +2,38 @@
   "command": "/usr/bin/neutron-server --config-file /usr/share/neutron/neutron-dist.conf --config-file /etc/neutron/neutron.conf --config-dir /etc/neutron/neutron.conf.d",
   "config_files": [
     {
-      "source": "/var/lib/config-data/01-neutron.conf",
+      "source": "/var/lib/config-data/default/01-neutron.conf",
       "dest": "/etc/neutron/neutron.conf.d/01-neutron.conf",
       "owner": "root:neutron",
       "perm": "0640"
     },
     {
-      "source": "/var/lib/config-data/02-neutron-custom.conf",
+      "source": "/var/lib/config-data/default/02-neutron-custom.conf",
       "dest": "/etc/neutron/neutron.conf.d/02-neutron-custom.conf",
       "owner": "root:neutron",
       "perm": "0640"
     },
     {
-      "source": "/var/lib/config-data/my.cnf",
+      "source": "/var/lib/config-data/default/my.cnf",
       "dest": "/etc/my.cnf",
       "owner": "neutron",
       "perm": "0644"
+    },
+    {
+      "source": "/var/lib/config-data/tls/certs/*",
+      "dest": "/etc/pki/tls/certs/",
+      "owner": "root:neutron",
+      "perm": "0640",
+      "optional": true,
+      "merge": true
+    },
+    {
+      "source": "/var/lib/config-data/tls/private/*",
+      "dest": "/etc/pki/tls/private/",
+      "owner": "root:neutron",
+      "perm": "0640",
+      "optional": true,
+      "merge": true
     }
   ]
 }

templates/ovn-agent.conf

@@ -1,3 +1,11 @@
 [ovn]
 ovn_nb_connection = {{ .NBConnection }}
 ovn_sb_connection = {{ .SBConnection }}
+{{- if .OVNDB_TLS }}
+ovn_nb_private_key = /etc/pki/tls/private/ovndb.key
+ovn_nb_certificate = /etc/pki/tls/certs/ovndb.crt
+ovn_nb_ca_cert = /etc/pki/tls/certs/ovndbca.crt
+ovn_sb_private_key = /etc/pki/tls/private/ovndb.key
+ovn_sb_certificate = /etc/pki/tls/certs/ovndb.crt
+ovn_sb_ca_cert = /etc/pki/tls/certs/ovndbca.crt
+{{- end }}

templates/ovn-metadata-agent.conf

@@ -6,3 +6,8 @@
 
 [ovn]
 ovn_sb_connection = {{ .SBConnection }}
+{{- if .OVNDB_TLS }}
+ovn_sb_private_key = /etc/pki/tls/private/ovndb.key
+ovn_sb_certificate = /etc/pki/tls/certs/ovndb.crt
+ovn_sb_ca_cert = /etc/pki/tls/certs/ovndbca.crt
+{{- end }}

test/functional/base_test.go

@@ -38,6 +38,7 @@ const (
 	PublicCertSecretName   = "public-tls-certs"
 	InternalCertSecretName = "internal-tls-certs"
 	CABundleSecretName     = "combined-ca-bundle"
+	OvnDbCertSecretName    = "ovndb-tls-certs"
 
 	timeout  = time.Second * 10
 	interval = timeout / 100

test/functional/neutronapi_controller_test.go

@@ -51,6 +51,7 @@ var _ = Describe("NeutronAPI controller", func() {
 	var caBundleSecretName types.NamespacedName
 	var internalCertSecretName types.NamespacedName
 	var publicCertSecretName types.NamespacedName
+	var ovnDbCertSecretName types.NamespacedName
 
 	BeforeEach(func() {
 		name = fmt.Sprintf("neutron-%s", uuid.New().String())
@@ -84,6 +85,10 @@ var _ = Describe("NeutronAPI controller", func() {
 			Name:      PublicCertSecretName,
 			Namespace: namespace,
 		}
+		ovnDbCertSecretName = types.NamespacedName{
+			Name:      OvnDbCertSecretName,
+			Namespace: namespace,
+		}
 	})
 
 	When("A NeutronAPI instance is created", func() {
@@ -1013,6 +1018,9 @@ var _ = Describe("NeutronAPI controller", func() {
 					},
 				},
 				"caBundleSecretName": CABundleSecretName,
+				"ovndb": map[string]interface{}{
+					"secretName": InternalCertSecretName,
+				},
 			}
 			DeferCleanup(th.DeleteInstance, CreateNeutronAPI(neutronAPIName.Namespace, neutronAPIName.Name, spec))
 
@@ -1055,10 +1063,14 @@ var _ = Describe("NeutronAPI controller", func() {
 			th.AssertVolumeExists(caBundleSecretName.Name, deployment.Spec.Template.Spec.Volumes)
 			th.AssertVolumeExists(internalCertSecretName.Name, deployment.Spec.Template.Spec.Volumes)
 			th.AssertVolumeExists(publicCertSecretName.Name, deployment.Spec.Template.Spec.Volumes)
+			th.AssertVolumeExists(ovnDbCertSecretName.Name, deployment.Spec.Template.Spec.Volumes)
 
 			// svc container ca cert
 			nSvcContainer := deployment.Spec.Template.Spec.Containers[0]
 			th.AssertVolumeMountExists(caBundleSecretName.Name, "tls-ca-bundle.pem", nSvcContainer.VolumeMounts)
+			th.AssertVolumeMountExists(ovnDbCertSecretName.Name, "tls.key", nSvcContainer.VolumeMounts)
+			th.AssertVolumeMountExists(ovnDbCertSecretName.Name, "tls.crt", nSvcContainer.VolumeMounts)
+			th.AssertVolumeMountExists(ovnDbCertSecretName.Name, "ca.crt", nSvcContainer.VolumeMounts)
 
 			// httpd container certs
 			nHttpdProxyContainer := deployment.Spec.Template.Spec.Containers[1]
@@ -1087,7 +1099,14 @@ var _ = Describe("NeutronAPI controller", func() {
 				ContainSubstring("mysql_wsrep_sync_wait = 1"))
 			Expect(conf).Should(
 				ContainSubstring(fmt.Sprintf("connection=mysql+pymysql://neutron:12345678@hostname-for-test-neutron-db-instance.%s.svc/neutron?read_default_file=/etc/my.cnf", namespace)))
-
+			Expect(conf).Should(And(
+				ContainSubstring("ovn_nb_private_key = /etc/pki/tls/private/ovndb.key"),
+				ContainSubstring("ovn_nb_certificate = /etc/pki/tls/certs/ovndb.crt"),
+				ContainSubstring("ovn_nb_ca_cert = /etc/pki/tls/certs/ovndbca.crt"),
+				ContainSubstring("ovn_sb_private_key = /etc/pki/tls/private/ovndb.key"),
+				ContainSubstring("ovn_sb_certificate = /etc/pki/tls/certs/ovndb.crt"),
+				ContainSubstring("ovn_sb_ca_cert = /etc/pki/tls/certs/ovndbca.crt"),
+			))
 			conf = string(configData.Data["my.cnf"])
 			Expect(conf).To(
 				ContainSubstring("[client]\nssl-ca=/etc/pki/ca-trust/extracted/pem/tls-ca-bundle.pem\nssl=1"))

test/kuttl/tests/neutron_tls/01-assert.yaml

@@ -36,7 +36,7 @@ spec:
         - -c
         - /usr/local/bin/kolla_start
         volumeMounts:
-        - mountPath: /var/lib/config-data
+        - mountPath: /var/lib/config-data/default
           name: config
           readOnly: true
         - mountPath: /var/lib/kolla/config_files/config.json