[refinery] Add option to run refinery as a statefulset

charts/refinery/templates/_pod.tpl

@@ -0,0 +1,129 @@
+{{- define "refinery.pod" -}}
+metadata:
+  annotations:
+    checksum/config: {{ include (print $.Template.BasePath "/configmap-config.yaml") . | sha256sum }}
+    {{- with .Values.podAnnotations }}
+    {{- toYaml . | nindent 4 }}
+    {{- end }}
+    {{- if .Values.config.PrometheusMetrics.Enabled }}
+    prometheus.io/port: "9090"
+    prometheus.io/scrape: "true"
+    {{- end }}
+    {{- if not .Values.LiveReload }}
+    checksum/rules: {{ include (print $.Template.BasePath "/configmap-rules.yaml") . | sha256sum }}
+    {{- end }}
+  labels:
+  {{- include "refinery.selectorLabels" . | nindent 4 }}
+  {{- with .Values.podLabels }}
+  {{- toYaml . | nindent 4 }}
+  {{- end }}
+spec:
+  {{- with .Values.imagePullSecrets }}
+  imagePullSecrets:
+  {{- toYaml . | nindent 4 }}
+  {{- end }}
+  serviceAccountName: {{ include "refinery.serviceAccountName" . }}
+  {{- if eq .Values.mode "statefulset" }}
+  # The makes the pod hostnames be resolvable.
+  setHostnameAsFQDN: true
+  {{- end }}
+  securityContext:
+  {{- toYaml .Values.podSecurityContext | nindent 4 }}
+  containers:
+    - name: {{ .Chart.Name }}
+      securityContext:
+      {{- toYaml .Values.securityContext | nindent 8 }}
+      image: "{{ .Values.image.repository }}:{{ .Values.image.tag | default .Chart.AppVersion }}"
+      imagePullPolicy: {{ .Values.image.pullPolicy }}
+      command:
+        - refinery
+        - -c
+        - /etc/refinery/config.yaml
+        - -r
+        - /etc/refinery/rules.yaml
+        {{- if .Values.debug.enabled }}
+        - -d
+        {{- end }}
+        {{- range .Values.extraCommandArgs }}
+        - {{ . }}
+        {{- end }}
+      {{- with .Values.environment }}
+      env:
+      {{- toYaml . | nindent 8 }}
+      {{- end }}
+      ports:
+        - name: data
+          containerPort: 8080
+          protocol: TCP
+        - name: grpc
+          containerPort: 4317
+          protocol: TCP
+        - name: peer
+          containerPort: 8081
+          protocol: TCP
+        {{- if .Values.config.PrometheusMetrics.Enabled }}
+        - name: metrics
+          containerPort: 9090
+          protocol: TCP
+        {{- end }}
+        {{- if .Values.debug.enabled }}
+        - name: debug
+          containerPort: {{ .Values.debug.port }}
+          protocol: TCP
+        {{- end }}
+      volumeMounts:
+        - name: refinery-config
+          mountPath: /etc/refinery/
+        {{- with .Values.extraVolumeMounts }}
+        {{- toYaml . | nindent 8 }}
+        {{- end }}
+      livenessProbe:
+        httpGet:
+          path: /alive
+          port: data
+        initialDelaySeconds: 10
+        periodSeconds: 10
+        failureThreshold: 3
+      readinessProbe:
+        httpGet:
+          path: /alive
+          port: data
+        initialDelaySeconds: 0
+        periodSeconds: 3
+        failureThreshold: 5
+      resources:
+  {{- toYaml .Values.resources | nindent 8 }}
+  volumes:
+    - name: refinery-config
+      projected:
+        sources:
+          - configMap:
+              name: {{ include "refinery.fullname" . }}-config
+              items:
+                - key: config.yaml
+                  path: config.yaml
+          - configMap:
+            {{- if .Values.RulesConfigMapName }}
+              name: {{ .Values.RulesConfigMapName }}
+            {{- else }}
+              name: {{ include "refinery.fullname" . }}-rules
+            {{- end }}
+              items:
+                - key: rules.yaml
+                  path: rules.yaml
+    {{- with .Values.extraVolumes }}
+    {{- toYaml . | nindent 4 }}
+    {{- end }}
+  {{- with .Values.nodeSelector }}
+  nodeSelector:
+  {{- toYaml . | nindent 4 }}
+  {{- end }}
+  {{- with .Values.affinity }}
+  affinity:
+  {{- toYaml . | nindent 4 }}
+  {{- end }}
+  {{- with .Values.tolerations }}
+  tolerations:
+  {{- toYaml . | nindent 4 }}
+  {{- end }}
+{{- end }}

charts/refinery/templates/deployment.yaml

@@ -1,3 +1,4 @@
+{{- if eq .Values.mode "deployment" -}}
 apiVersion: apps/v1
 kind: Deployment
 metadata:
@@ -16,126 +17,5 @@ spec:
     matchLabels:
   {{- include "refinery.selectorLabels" . | nindent 6 }}
   template:
-    metadata:
-      annotations:
-        checksum/config: {{ include (print $.Template.BasePath "/configmap-config.yaml") . | sha256sum }}
-        {{- with .Values.podAnnotations }}
-        {{- toYaml . | nindent 8 }}
-        {{- end }}
-        {{- if .Values.config.PrometheusMetrics.Enabled }}
-        prometheus.io/port: "9090"
-        prometheus.io/scrape: "true"
-        {{- end }}
-        {{- if not .Values.LiveReload }}
-        checksum/rules: {{ include (print $.Template.BasePath "/configmap-rules.yaml") . | sha256sum }}
-        {{- end }}
-      labels:
-      {{- include "refinery.selectorLabels" . | nindent 8 }}
-      {{- with .Values.podLabels }}
-      {{- toYaml . | nindent 8 }}
-      {{- end }}
-    spec:
-      {{- with .Values.imagePullSecrets }}
-      imagePullSecrets:
-      {{- toYaml . | nindent 8 }}
-      {{- end }}
-      serviceAccountName: {{ include "refinery.serviceAccountName" . }}
-      securityContext:
-      {{- toYaml .Values.podSecurityContext | nindent 8 }}
-      containers:
-        - name: {{ .Chart.Name }}
-          securityContext:
-          {{- toYaml .Values.securityContext | nindent 12 }}
-          image: "{{ .Values.image.repository }}:{{ .Values.image.tag | default .Chart.AppVersion }}"
-          imagePullPolicy: {{ .Values.image.pullPolicy }}
-          command:
-            - refinery
-            - -c
-            - /etc/refinery/config.yaml
-            - -r
-            - /etc/refinery/rules.yaml
-            {{- if .Values.debug.enabled }}
-            - -d
-            {{- end }}
-            {{- range .Values.extraCommandArgs }}
-            - {{ . }}
-            {{- end }}
-          {{- with .Values.environment }}
-          env:
-          {{- toYaml . | nindent 12 }}
-          {{- end }}
-          ports:
-            - name: data
-              containerPort: 8080
-              protocol: TCP
-            - name: grpc
-              containerPort: 4317
-              protocol: TCP
-            - name: peer
-              containerPort: 8081
-              protocol: TCP
-            {{- if .Values.config.PrometheusMetrics.Enabled }}
-            - name: metrics
-              containerPort: 9090
-              protocol: TCP
-            {{- end }}
-            {{- if .Values.debug.enabled }}
-            - name: debug
-              containerPort: {{ .Values.debug.port }}
-              protocol: TCP
-            {{- end }}
-          volumeMounts:
-            - name: refinery-config
-              mountPath: /etc/refinery/
-            {{- with .Values.extraVolumeMounts }}
-            {{- toYaml . | nindent 12 }}
-            {{- end }}
-          livenessProbe:
-            httpGet:
-              path: /alive
-              port: data
-            initialDelaySeconds: 10
-            periodSeconds: 10
-            failureThreshold: 3
-          readinessProbe:
-            httpGet:
-              path: /alive
-              port: data
-            initialDelaySeconds: 0
-            periodSeconds: 3
-            failureThreshold: 5
-          resources:
-      {{- toYaml .Values.resources | nindent 12 }}
-      volumes:
-        - name: refinery-config
-          projected:
-            sources:
-              - configMap:
-                  name: {{ include "refinery.fullname" . }}-config
-                  items:
-                    - key: config.yaml
-                      path: config.yaml
-              - configMap:
-                {{- if .Values.RulesConfigMapName }}
-                  name: {{ .Values.RulesConfigMapName }}
-                {{- else }}
-                  name: {{ include "refinery.fullname" . }}-rules
-                {{- end }}
-                  items:
-                    - key: rules.yaml
-                      path: rules.yaml
-        {{- with .Values.extraVolumes }}
-        {{- toYaml . | nindent 8 }}
-        {{- end }}
-      {{- with .Values.nodeSelector }}
-      nodeSelector:
-      {{- toYaml . | nindent 8 }}
-      {{- end }}
-      {{- with .Values.affinity }}
-      affinity:
-      {{- toYaml . | nindent 8 }}
-      {{- end }}
-      {{- with .Values.tolerations }}
-      tolerations:
-      {{- toYaml . | nindent 8 }}
-      {{- end }}
+  {{- include "refinery.pod" . | nindent 4 }}
+{{- end }}

charts/refinery/templates/headless-service.yaml

@@ -0,0 +1,19 @@
+{{- if eq .Values.mode "statefulset" -}}
+# Governing service to provide stable network ID for StatefulSet pods:
+#   https://kubernetes.io/docs/concepts/workloads/controllers/statefulset/#stable-network-id
+apiVersion: v1
+kind: Service
+metadata:
+  name: {{ include "refinery.fullname" . }}-cluster
+  namespace: {{ .Release.Namespace }}
+  labels:
+    {{- include "refinery.labels" . | nindent 4 }}
+    {{- with .Values.service.labels }}
+    {{- toYaml . | nindent 4 }}
+    {{- end }}
+spec:
+  type: ClusterIP
+  clusterIP: None
+  selector:
+    {{- include "refinery.selectorLabels" . | nindent 4 }}
+{{- end }}

charts/refinery/templates/hpa.yaml

@@ -9,7 +9,7 @@ metadata:
 spec:
   scaleTargetRef:
     apiVersion: apps/v1
-    kind: Deployment
+    kind: {{ if eq .Values.mode "statefulset" }}StatefulSet{{ else }}Deployment{{ end }}
     name: {{ include "refinery.fullname" . }}
   minReplicas: {{ .Values.autoscaling.minReplicas }}
   maxReplicas: {{ .Values.autoscaling.maxReplicas }}

charts/refinery/templates/statefulset.yaml

@@ -0,0 +1,33 @@
+{{- if eq .Values.mode "statefulset" -}}
+apiVersion: apps/v1
+kind: StatefulSet
+metadata:
+  name: {{ include "refinery.fullname" . }}
+  namespace: {{ .Release.Namespace }}
+  labels:
+  {{- include "refinery.labels" . | nindent 4 }}
+{{- with .Values.deploymentAnnotations }}
+  annotations: {{ toYaml . | nindent 4 }}
+{{- end }}
+spec:
+  # Governing service to provide stable network ID for StatefulSet pods:
+  #   https://kubernetes.io/docs/concepts/workloads/controllers/statefulset/#stable-network-id
+  serviceName: {{ include "refinery.fullname" . }}-cluster
+  # Refinery doesn't really have any state, so in theory there's no
+  # need for the controlled scale-up / scale-down of the default
+  # OrderedReady policy. However if all the pods come up at once while
+  # the redis peer list exists, most pods will crash loop because
+  # they're unable to reach some of the peers. The OrderedReady delay
+  # gives time for membership to expire, and makes it quicker overall
+  # unless you also take care to blow away the redis peer list when
+  # scaling.
+  podManagementPolicy: OrderedReady
+  {{- if not .Values.autoscaling.enabled }}
+  replicas: {{ .Values.replicaCount }}
+  {{- end }}
+  selector:
+    matchLabels:
+  {{- include "refinery.selectorLabels" . | nindent 6 }}
+  template:
+  {{- include "refinery.pod" . | nindent 4 }}
+{{- end }}

charts/refinery/values.schema.json

@@ -3,6 +3,11 @@
   "type": "object",
   "title": "Values",
   "properties": {
+    "mode": {
+      "type": "string",
+      "enum": ["deployment", "statefulset"],
+      "description": "Deployment mode"
+    },
     "config": {
       "type": "object",
       "additionalProperties": false,

charts/refinery/values.yaml

@@ -1,5 +1,13 @@
 # Default values for refinery.
 
+# Deployment mode
+#
+# Set to either `deployment` or `statefulset`. The default is `deployment`, but
+# `statefulset` might be preferred because the refinery nodes have stable
+# hostnames. This means the cluster membership appears static even if individual
+# pods are rescheduled.
+mode: "deployment"
+
 # The Refinery configuration.
 # This section allows setting all Refinery configuration options.
 # Configuration values set here by default have been set based on the requirements of the chart and Kubernetes.
@@ -14,10 +22,14 @@ config:
 
   PeerManagement:
     Type: redis
-    # IdentifierInterfaceName specifies a network interface to use when finding a local hostname.
-    # Due to the nature of DNS in Kubernetes, it is recommended to set this value to the 'eth0' interface name.
-    # When configured the pod's IP will be used in the peer list
-    IdentifierInterfaceName: eth0
+    # When running as a deployment, pod names are not stable or resolvable, so
+    # it is recommended to set this value to the 'eth0' interface name. This
+    # way, refinery will use the pod's IP peer list.
+    #
+    # When running as a statefulset, pods will have a stable and resolvable DNS
+    # name, so we can leave this empty and rely on the default which calls
+    # `os.Hostname()`.
+    IdentifierInterfaceName: "{{ if eq .Values.mode \"statefulset\" }}{{ else }}eth0{{ end }}"
 
   RedisPeerManagement:
     # Host is the host and port of the Redis instance to use for peer cluster membership management.