[nova] Add ProxySQL side-cars

Since nova stores & reads the db url from the database, one cannot simply set proxysql.mode=unix_socket, as first all pods need to have a proxysql sidecar. First one needs to roll-out with proxysql.mode=host_alias and after all pods with db access have a proxysql sidecar, one can switch with second deployment to unix_socket
sapcc · Jul 13, 2022 · 4638d80 · 4638d80
1 parent 8f377f4
commit 4638d80
Show file tree

Hide file tree

Showing 21 changed files with 196 additions and 49 deletions.
diff --git a/openstack/nova/requirements.lock b/openstack/nova/requirements.lock
@@ -1,10 +1,10 @@
 dependencies:
 - name: mariadb
   repository: https://charts.eu-de-2.cloud.sap
-  version: 0.3.53
+  version: 0.4.0
 - name: mariadb
   repository: https://charts.eu-de-2.cloud.sap
-  version: 0.3.53
+  version: 0.4.0
 - name: mysql_metrics
   repository: https://charts.eu-de-2.cloud.sap
   version: 0.2.7
@@ -22,12 +22,12 @@ dependencies:
   version: 0.4.2
 - name: mariadb
   repository: https://charts.eu-de-2.cloud.sap
-  version: 0.3.53
+  version: 0.4.0
 - name: rabbitmq
   repository: https://charts.eu-de-2.cloud.sap
   version: 0.4.0
 - name: region_check
   repository: https://charts.eu-de-2.cloud.sap
   version: 0.1.2
-digest: sha256:a1b2317c7f0d385e8cd361a6b2bd96b2bf273183419723e8a421b1508cbecf2f
-generated: "2022-07-12T12:11:48.511230569+02:00"
+digest: sha256:a84ac248a39a3048cd04f3c84fa3f6824a9522ea836fd947d741b37f8ad0f433
+generated: "2022-07-13T09:41:34.423009048+02:00"
diff --git a/openstack/nova/requirements.yaml b/openstack/nova/requirements.yaml
@@ -2,12 +2,12 @@ dependencies:
   - condition: mariadb.enabled
     name: mariadb
     repository: https://charts.eu-de-2.cloud.sap
-    version: 0.3.53
+    version: 0.4.0
   - name: mariadb
     alias: mariadb_api
     condition: mariadb_api.enabled
     repository: https://charts.eu-de-2.cloud.sap
-    version: 0.3.53
+    version: 0.4.0
   - condition: mariadb.enabled
     name: mysql_metrics
     repository: https://charts.eu-de-2.cloud.sap
@@ -30,7 +30,7 @@ dependencies:
     alias: mariadb_cell2
     condition: mariadb_cell2.enabled
     repository: https://charts.eu-de-2.cloud.sap
-    version: 0.3.53
+    version: 0.4.0
   - name: rabbitmq
     alias: rabbitmq_cell2
     condition: cell2.enabled

diff --git a/openstack/nova/templates/_console_deployment.yaml.tpl b/openstack/nova/templates/_console_deployment.yaml.tpl
@@ -31,15 +31,21 @@ spec:
 {{ tuple . "nova" (print "console-" $name) | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 8 }}
       annotations:
         configmap-etc-hash: {{ include (print .Template.BasePath "/etc-configmap.yaml") . | sha256sum }}
+        {{- if .Values.proxysql.mode }}
+        prometheus.io/scrape: "true"
+        prometheus.io/targets: {{ required ".Values.alerts.prometheus missing" .Values.alerts.prometheus | quote }}
+        {{- end }}
     spec:
 {{ tuple . "nova" (print "console-" $name) | include "kubernetes_pod_anti_affinity" | indent 6 }}
+{{ include "utils.proxysql.pod_settings" . | indent 6 }}
       hostname: nova-console-{{ $name }}
       volumes:
       - name: etcnova
         emptyDir: {}
       - name: nova-etc
         configMap:
           name: nova-etc
+      {{- include "utils.proxysql.volumes" . | indent 6 }}
       containers:
       - name: nova-console-{{ $name }}
         image: {{ required ".Values.global.registry is missing" .Values.global.registry}}/ubuntu-source-nova-{{ $name }}proxy:{{index .Values (print "imageVersionNova" (title $name) "proxy") | default .Values.imageVersionNova | default .Values.imageVersion | required "Please set nova.imageVersion or similar" }}
@@ -82,5 +88,7 @@ spec:
           mountPath: /etc/nova/logging.ini
           subPath: logging.ini
           readOnly: true
+        {{- include "utils.proxysql.volume_mount" . | indent 8 }}
+      {{- include "utils.proxysql.container" . | indent 6 }}
 {{- end }}
 {{- end }}
diff --git a/openstack/nova/templates/_helpers.tpl b/openstack/nova/templates/_helpers.tpl
@@ -1,12 +1,12 @@
-{{- define "cell0_db_path" -}}
-mysql+pymysql://{{.Values.cell0dbUser}}:{{ default .Values.cell0dbPassword .Values.global.dbPassword | urlquery }}@{{.Chart.Name}}-mariadb.{{include "svc_fqdn" .}}:3306/{{.Values.cell0dbName}}?charset=utf8
-{{- end -}}
+{{- define "cell0_db_path" }}
+    {{- tuple . .Values.cell0dbName .Values.cell0dbUser (default .Values.cell0dbPassword .Values.global.dbPassword) | include "db_url_mysql" }}
+{{- end }}
 
 {{- define "cell2_db_path" -}}
-{{- if eq .Values.cell2.enabled true -}}
-mysql+pymysql://{{.Values.cell2dbUser}}:{{ default .Values.cell2dbPassword .Values.global.dbPassword | urlquery }}@{{.Chart.Name}}-{{.Values.cell2.name}}-mariadb.{{include "svc_fqdn" .}}:3306/{{.Values.cell2dbName}}?charset=utf8
-{{- end -}}
-{{- end -}}
+    {{- if eq .Values.cell2.enabled true -}}
+        {{- tuple . .Values.cell2dbName .Values.cell2dbUser (default .Values.cell2dbPassword .Values.global.dbPassword) .Values.mariadb_cell2.name | include "db_url_mysql" }}
+    {{- end }}
+{{- end }}
 
 {{- define "cell2_db_path_for_exporter" -}}
 {{- if eq .Values.cell2.enabled true -}}

diff --git a/openstack/nova/templates/api-deployment.yaml b/openstack/nova/templates/api-deployment.yaml
@@ -29,13 +29,18 @@ spec:
 {{ tuple . "nova" "api" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 8 }}
       annotations:
         configmap-etc-hash: {{ include (print $.Template.BasePath "/etc-configmap.yaml") . | sha256sum }}
+        {{- if or .Values.conductor.config_file.DEFAULT.statsd_enabled .Values.proxysql.mode }}
+        prometheus.io/scrape: "true"
+        prometheus.io/targets: {{ required ".Values.alerts.prometheus missing" .Values.alerts.prometheus | quote }}
+        {{- end }}
     spec:
       terminationGracePeriodSeconds: {{ .Values.defaults.default.graceful_shutdown_timeout | add 5 }}
 {{- if .Values.pod.debug.api }}
       securityContext:
         runAsUser: 0
 {{- end }}
 {{ tuple . "nova" "api" | include "kubernetes_pod_anti_affinity" | indent 6 }}
+{{ include "utils.proxysql.pod_settings" . | indent 6 }}
       hostname: nova-api
       containers:
         - name: nova-api
@@ -149,6 +154,8 @@ spec:
               subPath: watcher.yaml
               readOnly: true
             {{- end }}
+            {{- include "utils.proxysql.volume_mount" . | indent 12 }}
+        {{- include "utils.proxysql.container" . | indent 8 }}
         - name: statsd
           image: {{ required ".Values.global.dockerHubMirror is missing" .Values.global.dockerHubMirror}}/prom/statsd-exporter:v0.8.1
           imagePullPolicy: IfNotPresent
@@ -174,3 +181,4 @@ spec:
           configMap:
             name: nova-bin
             defaultMode: 0755
+        {{- include "utils.proxysql.volumes" . | indent 8 }}
diff --git a/openstack/nova/templates/api-metadata-deployment.yaml b/openstack/nova/templates/api-metadata-deployment.yaml
@@ -29,9 +29,14 @@ spec:
 {{ tuple . "nova" "api-metadata" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 8 }}
       annotations:
         configmap-etc-hash: {{ include (print .Template.BasePath "/etc-configmap.yaml") . | sha256sum }}
+        {{- if or .Values.conductor.config_file.DEFAULT.statsd_enabled .Values.proxysql.mode }}
+        prometheus.io/scrape: "true"
+        prometheus.io/targets: {{ required ".Values.alerts.prometheus missing" .Values.alerts.prometheus | quote }}
+        {{- end }}
     spec:
-      terminationGracePeriodSeconds: {{ .Values.defaults.default.graceful_shutdown_timeout | add 5 }}
 {{ tuple . "nova" "api-metadata" | include "kubernetes_pod_anti_affinity" | indent 6 }}
+{{ include "utils.proxysql.pod_settings" . | indent 6 }}
+      terminationGracePeriodSeconds: {{ .Values.defaults.default.graceful_shutdown_timeout | add 5 }}
       hostname: nova-api-metadata
       containers:
         - name: nova-api-metadata
@@ -137,6 +142,8 @@ spec:
               subPath: watcher.yaml
               readOnly: true
             {{- end }}
+            {{- include "utils.proxysql.volume_mount" . | indent 12 }}
+        {{- include "utils.proxysql.container" . | indent 8 }}
         - name: statsd
           image: {{ required ".Values.global.dockerHubMirror is missing" .Values.global.dockerHubMirror}}/prom/statsd-exporter:v0.8.1
           imagePullPolicy: IfNotPresent
@@ -162,3 +169,4 @@ spec:
           configMap:
             name: nova-bin
             defaultMode: 0755
+        {{- include "utils.proxysql.volumes" . | indent 8 }}
diff --git a/openstack/nova/templates/bigvm-deployment.yaml b/openstack/nova/templates/bigvm-deployment.yaml
@@ -30,8 +30,13 @@ spec:
 {{ tuple . "nova" "bigvm" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 8 }}
       annotations:
         configmap-etc-hash: {{ include (print $.Template.BasePath "/etc-configmap.yaml") . | sha256sum }}
+        {{- if or .Values.conductor.config_file.DEFAULT.statsd_enabled .Values.proxysql.mode }}
+        prometheus.io/scrape: "true"
+        prometheus.io/targets: {{ required ".Values.alerts.prometheus missing" .Values.alerts.prometheus | quote }}
+        {{- end }}
     spec:
 {{ tuple . "nova" "bigvm" | include "kubernetes_pod_anti_affinity" | indent 6 }}
+{{ include "utils.proxysql.pod_settings" . | indent 6 }}
       terminationGracePeriodSeconds: {{ .Values.defaults.default.graceful_shutdown_timeout | add 5 }}
       hostname: nova-bigvm
       containers:
@@ -84,10 +89,13 @@ spec:
               mountPath: /etc/nova/logging.ini
               subPath: logging.ini
               readOnly: true
+            {{- include "utils.proxysql.volume_mount" . | indent 12 }}
+        {{- include "utils.proxysql.container" . | indent 8 }}
       volumes:
         - name: etcnova
           emptyDir: {}
         - name: nova-etc
           configMap:
             name: nova-etc
+        {{- include "utils.proxysql.volumes" . | indent 8 }}
 {{- end }}
diff --git a/openstack/nova/templates/bin/_db-migrate.tpl b/openstack/nova/templates/bin/_db-migrate.tpl
@@ -1,9 +1,10 @@
-#!/usr/bin/env bash
-set -e
-set -x
+#!/bin/bash
+set -exuo pipefail
 
 nova-manage api_db sync
 nova-manage db sync
 nova-manage db null_instance_uuid_scan --delete
 
 # online data migration run by online-migration-job
+
+{{ include "utils.proxysql.proxysql_signal_stop_script" . }}
diff --git a/openstack/nova/templates/bin/_db-online-migrate.tpl b/openstack/nova/templates/bin/_db-online-migrate.tpl
@@ -1,7 +1,5 @@
-#!/bin/sh
-
-set -x
-set -e
+#!/bin/bash
+set -exuo pipefail
 
 nova_manage="nova-manage --config-file /etc/nova/nova.conf"
 available_commands_text=$(nova-manage --help | awk '/Command categories/ {getline; print $0}')
@@ -14,3 +12,5 @@ $nova_manage db online_data_migrations
 if echo "${available_commands_text}" | grep -q -E '[{,]placement[},]'; then
   $nova_manage placement sync_aggregates
 fi
+
+{{ include "utils.proxysql.proxysql_signal_stop_script" . }}
diff --git a/openstack/nova/templates/bin/_db-update-cells.tpl b/openstack/nova/templates/bin/_db-update-cells.tpl
@@ -1,7 +1,5 @@
 #!/bin/bash
-
-set -e
-
+set -euo pipefail
 
 update_cell() {
   cell_name="${1}"; shift
@@ -57,7 +55,7 @@ for line in $(nova-manage cell_v2 list_cells --verbose | grep ':/'); do
       found_cell1="true"
       update_cell "${cell_name}" "${cell_uuid}" \
             "${transport_url}" "{{ tuple . .Values.rabbitmq | include "rabbitmq._transport_url" }}" \
-            "${database_connection}" "{{ tuple . .Values.dbName .Values.dbUser (default .Values.dbPassword .Values.global.dbPassword) | include "db_url_mysql" }}"
+            "${database_connection}" "{{ include "db_url_mysql" . }}"
       ;;
 {{ if .Values.cell2.enabled }}
     {{.Values.cell2.name}})
@@ -81,7 +79,7 @@ if [ "${found_cell1}" = "false" ]; then
   nova-manage cell_v2 create_cell --verbose \
       --name "cell1" \
       --transport-url "{{ tuple . .Values.rabbitmq | include "rabbitmq._transport_url" }}" \
-      --database_connection "{{ tuple . .Values.dbName .Values.dbUser (default .Values.dbPassword .Values.global.dbPassword) | include "db_url_mysql" }}"
+      --database_connection "{{ include "db_url_mysql" . }}"
   nova-manage cell_v2 discover_hosts
 fi
 
@@ -95,4 +93,4 @@ if [ "$found_cell2" = "false" ]; then
 fi
 {{- end }}
 
-exit
+{{ include "utils.proxysql.proxysql_signal_stop_script" . }}
diff --git a/openstack/nova/templates/cell2-conductor-deployment.yaml b/openstack/nova/templates/cell2-conductor-deployment.yaml
@@ -29,40 +29,44 @@ spec:
         alert-service: nova
 {{ tuple . "nova" "conductor" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 8 }}
       annotations:
-        {{- if .Values.cell2.conductor.config_file.DEFAULT.statsd_enabled }}
+        {{- if or .Values.cell2.conductor.config_file.DEFAULT.statsd_enabled .Values.proxysql.mode }}
         prometheus.io/scrape: "true"
         prometheus.io/targets: {{ required ".Values.alerts.prometheus missing" .Values.alerts.prometheus | quote }}
         {{- end }}
         configmap-etc-hash: {{ include (print $.Template.BasePath "/etc-configmap.yaml") . | sha256sum }}
     spec:
 {{ tuple . "nova" "conductor" | include "kubernetes_pod_anti_affinity" | indent 6 }}
+{{ include "utils.proxysql.pod_settings" . | indent 6 }}
       terminationGracePeriodSeconds: {{ .Values.defaults.default.graceful_shutdown_timeout | add 5 }}
       hostname: nova-conductor
       initContainers:
-        - name: nova-db-sync
-          image: {{ required ".Values.global.registry is missing" .Values.global.registry}}/ubuntu-source-nova-api:{{.Values.imageVersionNovaApi | default .Values.imageVersionNova | default .Values.imageVersion | required "Please set nova.imageVersion or similar"}}
-          command: ['sh', '-c', 'nova-manage db sync --local_cell']
-          volumeMounts:
-            - mountPath: /etc/nova
-              name: etcnova
-            - mountPath: /etc/nova/nova.conf
-              name: nova-etc
-              subPath: nova-cell2.conf
-              readOnly: true
-      containers:
-        - name: nova-conductor
+        - name: dependencies
           image: {{ required ".Values.global.registry is missing" .Values.global.registry}}/ubuntu-source-nova-conductor:{{.Values.imageVersionNovaConductor | default .Values.imageVersionNova | default .Values.imageVersion | required "Please set nova.imageVersion or similar"}}
           imagePullPolicy: IfNotPresent
           command:
-            - dumb-init
             - kubernetes-entrypoint
           env:
             - name: COMMAND
-              value: "nova-conductor --config-file etc/nova/nova.conf --config-file /etc/nova/nova-conductor.conf --config-file /etc/nova/nova-cell2.conf"
+              value: "true"
             - name: NAMESPACE
               value: {{ .Release.Namespace }}
             - name: DEPENDENCY_SERVICE
-              value: "{{ .Release.Name }}-{{ .Values.cell2.name }}-rabbitmq,{{ .Release.Name }}-{{ .Values.cell2.name }}-mariadb"
+              value: "{{ .Release.Name }}-{{ .Values.cell2.name }}-rabbitmq,{{ .Release.Name }}-{{ .Values.cell2.name }}-mariadb"      
+            - name: DEPENDENCY_JOB
+              value: "nova-update-cells2-{{ .Release.Revision }}"
+      containers:
+        - name: nova-conductor
+          image: {{ required ".Values.global.registry is missing" .Values.global.registry}}/ubuntu-source-nova-conductor:{{.Values.imageVersionNovaConductor | default .Values.imageVersionNova | default .Values.imageVersion | required "Please set nova.imageVersion or similar"}}
+          imagePullPolicy: IfNotPresent
+          command:
+            - nova-conductor
+            - --config-file
+            - /etc/nova/nova.conf
+            - --config-file
+            - /etc/nova/nova-conductor.conf
+            - --config-file
+            - /etc/nova/nova-cell2.conf
+          env:
             {{- if .Values.sentry.enabled }}
             - name: SENTRY_DSN
               valueFrom:
@@ -107,6 +111,8 @@ spec:
               name: nova-etc
               subPath: logging.ini
               readOnly: true
+            {{- include "utils.proxysql.volume_mount" . | indent 12 }}
+        {{- include "utils.proxysql.container" . | indent 8 }}
         {{- if .Values.cell2.conductor.config_file.DEFAULT.statsd_enabled }}
         - name: statsd
           image: {{ required ".Values.global.dockerHubMirror is missing" .Values.global.dockerHubMirror}}/prom/statsd-exporter:v0.8.1
@@ -130,4 +136,5 @@ spec:
         - name: nova-etc
           configMap:
             name: nova-etc
+        {{- include "utils.proxysql.volumes" . | indent 8 }}
 {{- end }}
diff --git a/openstack/nova/templates/conductor-deployment.yaml b/openstack/nova/templates/conductor-deployment.yaml
@@ -28,13 +28,14 @@ spec:
         alert-service: nova
 {{ tuple . "nova" "conductor" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 8 }}
       annotations:
-        {{- if .Values.conductor.config_file.DEFAULT.statsd_enabled }}
+        {{- if or .Values.conductor.config_file.DEFAULT.statsd_enabled .Values.proxysql.mode }}
         prometheus.io/scrape: "true"
         prometheus.io/targets: {{ required ".Values.alerts.prometheus missing" .Values.alerts.prometheus | quote }}
         {{- end }}
         configmap-etc-hash: {{ include (print $.Template.BasePath "/etc-configmap.yaml") . | sha256sum }}
     spec:
 {{ tuple . "nova" "conductor" | include "kubernetes_pod_anti_affinity" | indent 6 }}
+{{ include "utils.proxysql.pod_settings" . | indent 6 }}
       terminationGracePeriodSeconds: {{ .Values.defaults.default.graceful_shutdown_timeout | add 5 }}
       hostname: nova-conductor
       containers:
@@ -91,6 +92,8 @@ spec:
               name: nova-etc
               subPath: logging.ini
               readOnly: true
+            {{- include "utils.proxysql.volume_mount" . | indent 12 }}
+        {{- include "utils.proxysql.container" . | indent 8 }}
         {{- if .Values.conductor.config_file.DEFAULT.statsd_enabled }}
         - name: statsd
           image: {{ required ".Values.global.dockerHubMirror is missing" .Values.global.dockerHubMirror}}/prom/statsd-exporter:v0.8.1
@@ -114,3 +117,4 @@ spec:
         - name: nova-etc
           configMap:
             name: nova-etc
+        {{- include "utils.proxysql.volumes" . | indent 8 }}
diff --git a/openstack/nova/templates/consoleauth-deployment.yaml b/openstack/nova/templates/consoleauth-deployment.yaml
@@ -32,13 +32,15 @@ spec:
         configmap-etc-hash: {{ include (print $.Template.BasePath "/etc-configmap.yaml") . | sha256sum }}
     spec:
 {{ tuple . "nova" "console-auth" | include "kubernetes_pod_anti_affinity" | indent 6 }}
+{{ include "utils.proxysql.pod_settings" . | indent 6 }}
       hostname: nova-console
       volumes:
       - name: etcnova
         emptyDir: {}
       - name: nova-etc
         configMap:
           name: nova-etc
+      {{- include "utils.proxysql.volumes" . | indent 6 }}
       containers:
       - name: nova-consoleauth
         image: {{ required ".Values.global.registry is missing" .Values.global.registry}}/ubuntu-source-nova-consoleauth:{{.Values.imageVersionNovaConsoleauth |  default .Values.imageVersionNova | default .Values.imageVersion | required "Please set nova.imageVersion or similar" }}
@@ -75,6 +77,8 @@ spec:
           mountPath: /etc/nova/logging.ini
           subPath: logging.ini
           readOnly: true
+        {{- include "utils.proxysql.volume_mount" . | indent 8 }}
+      {{- include "utils.proxysql.container" . | indent 6 }}
 {{- end }}
 {{ $envAll := . }}
 {{- range $name, $config := .Values.consoles }}

diff --git a/openstack/nova/templates/etc/_nova.conf.tpl b/openstack/nova/templates/etc/_nova.conf.tpl
@@ -41,7 +41,7 @@ dhcp_domain = openstack.{{ required ".Values.global.region is missing" .Values.g
 compute_link_prefix = https://{{include "nova_api_endpoint_host_public" .}}:{{.Values.global.novaApiPortPublic}}
 
 [api_database]
-connection = mysql+pymysql://{{.Values.apidbUser}}:{{.Values.apidbPassword | urlquery}}@nova-api-mariadb.{{.Release.Namespace}}.svc.kubernetes.{{.Values.global.region}}.{{.Values.global.tld}}/nova_api?charset=utf8
+connection = {{ tuple . .Values.apidbName .Values.apidbUser .Values.apidbPassword .Values.mariadb_api.name | include "db_url_mysql" }}
 {{- include "ini_sections.database_options_mysql" . }}
 
 {{ include "ini_sections.database" . }}