diff --git a/charts/refinery/templates/_pod.tpl b/charts/refinery/templates/_pod.tpl new file mode 100644 index 00000000..baea0a4a --- /dev/null +++ b/charts/refinery/templates/_pod.tpl @@ -0,0 +1,129 @@ +{{- define "refinery.pod" -}} +metadata: + annotations: + checksum/config: {{ include (print $.Template.BasePath "/configmap-config.yaml") . | sha256sum }} + {{- with .Values.podAnnotations }} + {{- toYaml . | nindent 4 }} + {{- end }} + {{- if .Values.config.PrometheusMetrics.Enabled }} + prometheus.io/port: "9090" + prometheus.io/scrape: "true" + {{- end }} + {{- if not .Values.LiveReload }} + checksum/rules: {{ include (print $.Template.BasePath "/configmap-rules.yaml") . | sha256sum }} + {{- end }} + labels: + {{- include "refinery.selectorLabels" . | nindent 4 }} + {{- with .Values.podLabels }} + {{- toYaml . | nindent 4 }} + {{- end }} +spec: + {{- with .Values.imagePullSecrets }} + imagePullSecrets: + {{- toYaml . | nindent 4 }} + {{- end }} + serviceAccountName: {{ include "refinery.serviceAccountName" . }} + {{- if eq .Values.mode "statefulset" }} + # The makes the pod hostnames be resolvable. + setHostnameAsFQDN: true + {{- end }} + securityContext: + {{- toYaml .Values.podSecurityContext | nindent 4 }} + containers: + - name: {{ .Chart.Name }} + securityContext: + {{- toYaml .Values.securityContext | nindent 8 }} + image: "{{ .Values.image.repository }}:{{ .Values.image.tag | default .Chart.AppVersion }}" + imagePullPolicy: {{ .Values.image.pullPolicy }} + command: + - refinery + - -c + - /etc/refinery/config.yaml + - -r + - /etc/refinery/rules.yaml + {{- if .Values.debug.enabled }} + - -d + {{- end }} + {{- range .Values.extraCommandArgs }} + - {{ . }} + {{- end }} + {{- with .Values.environment }} + env: + {{- toYaml . | nindent 8 }} + {{- end }} + ports: + - name: data + containerPort: 8080 + protocol: TCP + - name: grpc + containerPort: 4317 + protocol: TCP + - name: peer + containerPort: 8081 + protocol: TCP + {{- if .Values.config.PrometheusMetrics.Enabled }} + - name: metrics + containerPort: 9090 + protocol: TCP + {{- end }} + {{- if .Values.debug.enabled }} + - name: debug + containerPort: {{ .Values.debug.port }} + protocol: TCP + {{- end }} + volumeMounts: + - name: refinery-config + mountPath: /etc/refinery/ + {{- with .Values.extraVolumeMounts }} + {{- toYaml . | nindent 8 }} + {{- end }} + livenessProbe: + httpGet: + path: /alive + port: data + initialDelaySeconds: 10 + periodSeconds: 10 + failureThreshold: 3 + readinessProbe: + httpGet: + path: /alive + port: data + initialDelaySeconds: 0 + periodSeconds: 3 + failureThreshold: 5 + resources: + {{- toYaml .Values.resources | nindent 8 }} + volumes: + - name: refinery-config + projected: + sources: + - configMap: + name: {{ include "refinery.fullname" . }}-config + items: + - key: config.yaml + path: config.yaml + - configMap: + {{- if .Values.RulesConfigMapName }} + name: {{ .Values.RulesConfigMapName }} + {{- else }} + name: {{ include "refinery.fullname" . }}-rules + {{- end }} + items: + - key: rules.yaml + path: rules.yaml + {{- with .Values.extraVolumes }} + {{- toYaml . | nindent 4 }} + {{- end }} + {{- with .Values.nodeSelector }} + nodeSelector: + {{- toYaml . | nindent 4 }} + {{- end }} + {{- with .Values.affinity }} + affinity: + {{- toYaml . | nindent 4 }} + {{- end }} + {{- with .Values.tolerations }} + tolerations: + {{- toYaml . | nindent 4 }} + {{- end }} +{{- end }} diff --git a/charts/refinery/templates/deployment.yaml b/charts/refinery/templates/deployment.yaml index ab4b658d..7ae63584 100644 --- a/charts/refinery/templates/deployment.yaml +++ b/charts/refinery/templates/deployment.yaml @@ -1,3 +1,4 @@ +{{- if eq .Values.mode "deployment" -}} apiVersion: apps/v1 kind: Deployment metadata: @@ -16,126 +17,5 @@ spec: matchLabels: {{- include "refinery.selectorLabels" . | nindent 6 }} template: - metadata: - annotations: - checksum/config: {{ include (print $.Template.BasePath "/configmap-config.yaml") . | sha256sum }} - {{- with .Values.podAnnotations }} - {{- toYaml . | nindent 8 }} - {{- end }} - {{- if .Values.config.PrometheusMetrics.Enabled }} - prometheus.io/port: "9090" - prometheus.io/scrape: "true" - {{- end }} - {{- if not .Values.LiveReload }} - checksum/rules: {{ include (print $.Template.BasePath "/configmap-rules.yaml") . | sha256sum }} - {{- end }} - labels: - {{- include "refinery.selectorLabels" . | nindent 8 }} - {{- with .Values.podLabels }} - {{- toYaml . | nindent 8 }} - {{- end }} - spec: - {{- with .Values.imagePullSecrets }} - imagePullSecrets: - {{- toYaml . | nindent 8 }} - {{- end }} - serviceAccountName: {{ include "refinery.serviceAccountName" . }} - securityContext: - {{- toYaml .Values.podSecurityContext | nindent 8 }} - containers: - - name: {{ .Chart.Name }} - securityContext: - {{- toYaml .Values.securityContext | nindent 12 }} - image: "{{ .Values.image.repository }}:{{ .Values.image.tag | default .Chart.AppVersion }}" - imagePullPolicy: {{ .Values.image.pullPolicy }} - command: - - refinery - - -c - - /etc/refinery/config.yaml - - -r - - /etc/refinery/rules.yaml - {{- if .Values.debug.enabled }} - - -d - {{- end }} - {{- range .Values.extraCommandArgs }} - - {{ . }} - {{- end }} - {{- with .Values.environment }} - env: - {{- toYaml . | nindent 12 }} - {{- end }} - ports: - - name: data - containerPort: 8080 - protocol: TCP - - name: grpc - containerPort: 4317 - protocol: TCP - - name: peer - containerPort: 8081 - protocol: TCP - {{- if .Values.config.PrometheusMetrics.Enabled }} - - name: metrics - containerPort: 9090 - protocol: TCP - {{- end }} - {{- if .Values.debug.enabled }} - - name: debug - containerPort: {{ .Values.debug.port }} - protocol: TCP - {{- end }} - volumeMounts: - - name: refinery-config - mountPath: /etc/refinery/ - {{- with .Values.extraVolumeMounts }} - {{- toYaml . | nindent 12 }} - {{- end }} - livenessProbe: - httpGet: - path: /alive - port: data - initialDelaySeconds: 10 - periodSeconds: 10 - failureThreshold: 3 - readinessProbe: - httpGet: - path: /alive - port: data - initialDelaySeconds: 0 - periodSeconds: 3 - failureThreshold: 5 - resources: - {{- toYaml .Values.resources | nindent 12 }} - volumes: - - name: refinery-config - projected: - sources: - - configMap: - name: {{ include "refinery.fullname" . }}-config - items: - - key: config.yaml - path: config.yaml - - configMap: - {{- if .Values.RulesConfigMapName }} - name: {{ .Values.RulesConfigMapName }} - {{- else }} - name: {{ include "refinery.fullname" . }}-rules - {{- end }} - items: - - key: rules.yaml - path: rules.yaml - {{- with .Values.extraVolumes }} - {{- toYaml . | nindent 8 }} - {{- end }} - {{- with .Values.nodeSelector }} - nodeSelector: - {{- toYaml . | nindent 8 }} - {{- end }} - {{- with .Values.affinity }} - affinity: - {{- toYaml . | nindent 8 }} - {{- end }} - {{- with .Values.tolerations }} - tolerations: - {{- toYaml . | nindent 8 }} - {{- end }} + {{- include "refinery.pod" . | nindent 4 }} +{{- end }} diff --git a/charts/refinery/templates/headless-service.yaml b/charts/refinery/templates/headless-service.yaml new file mode 100644 index 00000000..ac486f27 --- /dev/null +++ b/charts/refinery/templates/headless-service.yaml @@ -0,0 +1,19 @@ +{{- if eq .Values.mode "statefulset" -}} +# Governing service to provide stable network ID for StatefulSet pods: +# https://kubernetes.io/docs/concepts/workloads/controllers/statefulset/#stable-network-id +apiVersion: v1 +kind: Service +metadata: + name: {{ include "refinery.fullname" . }}-cluster + namespace: {{ .Release.Namespace }} + labels: + {{- include "refinery.labels" . | nindent 4 }} + {{- with .Values.service.labels }} + {{- toYaml . | nindent 4 }} + {{- end }} +spec: + type: ClusterIP + clusterIP: None + selector: + {{- include "refinery.selectorLabels" . | nindent 4 }} +{{- end }} diff --git a/charts/refinery/templates/hpa.yaml b/charts/refinery/templates/hpa.yaml index bbc7b56f..06d2219b 100644 --- a/charts/refinery/templates/hpa.yaml +++ b/charts/refinery/templates/hpa.yaml @@ -9,7 +9,7 @@ metadata: spec: scaleTargetRef: apiVersion: apps/v1 - kind: Deployment + kind: {{ if eq .Values.mode "statefulset" }}StatefulSet{{ else }}Deployment{{ end }} name: {{ include "refinery.fullname" . }} minReplicas: {{ .Values.autoscaling.minReplicas }} maxReplicas: {{ .Values.autoscaling.maxReplicas }} diff --git a/charts/refinery/templates/statefulset.yaml b/charts/refinery/templates/statefulset.yaml new file mode 100644 index 00000000..9ba7ebe8 --- /dev/null +++ b/charts/refinery/templates/statefulset.yaml @@ -0,0 +1,33 @@ +{{- if eq .Values.mode "statefulset" -}} +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: {{ include "refinery.fullname" . }} + namespace: {{ .Release.Namespace }} + labels: + {{- include "refinery.labels" . | nindent 4 }} +{{- with .Values.deploymentAnnotations }} + annotations: {{ toYaml . | nindent 4 }} +{{- end }} +spec: + # Governing service to provide stable network ID for StatefulSet pods: + # https://kubernetes.io/docs/concepts/workloads/controllers/statefulset/#stable-network-id + serviceName: {{ include "refinery.fullname" . }}-cluster + # Refinery doesn't really have any state, so in theory there's no + # need for the controlled scale-up / scale-down of the default + # OrderedReady policy. However if all the pods come up at once while + # the redis peer list exists, most pods will crash loop because + # they're unable to reach some of the peers. The OrderedReady delay + # gives time for membership to expire, and makes it quicker overall + # unless you also take care to blow away the redis peer list when + # scaling. + podManagementPolicy: OrderedReady + {{- if not .Values.autoscaling.enabled }} + replicas: {{ .Values.replicaCount }} + {{- end }} + selector: + matchLabels: + {{- include "refinery.selectorLabels" . | nindent 6 }} + template: + {{- include "refinery.pod" . | nindent 4 }} +{{- end }} diff --git a/charts/refinery/values.schema.json b/charts/refinery/values.schema.json index ede8b910..54c13f22 100644 --- a/charts/refinery/values.schema.json +++ b/charts/refinery/values.schema.json @@ -3,6 +3,11 @@ "type": "object", "title": "Values", "properties": { + "mode": { + "type": "string", + "enum": ["deployment", "statefulset"], + "description": "Deployment mode" + }, "config": { "type": "object", "additionalProperties": false, diff --git a/charts/refinery/values.yaml b/charts/refinery/values.yaml index 6c8e7129..4e924c26 100644 --- a/charts/refinery/values.yaml +++ b/charts/refinery/values.yaml @@ -1,5 +1,13 @@ # Default values for refinery. +# Deployment mode +# +# Set to either `deployment` or `statefulset`. The default is `deployment`, but +# `statefulset` might be preferred because the refinery nodes have stable +# hostnames. This means the cluster membership appears static even if individual +# pods are rescheduled. +mode: "deployment" + # The Refinery configuration. # This section allows setting all Refinery configuration options. # Configuration values set here by default have been set based on the requirements of the chart and Kubernetes. @@ -14,10 +22,14 @@ config: PeerManagement: Type: redis - # IdentifierInterfaceName specifies a network interface to use when finding a local hostname. - # Due to the nature of DNS in Kubernetes, it is recommended to set this value to the 'eth0' interface name. - # When configured the pod's IP will be used in the peer list - IdentifierInterfaceName: eth0 + # When running as a deployment, pod names are not stable or resolvable, so + # it is recommended to set this value to the 'eth0' interface name. This + # way, refinery will use the pod's IP peer list. + # + # When running as a statefulset, pods will have a stable and resolvable DNS + # name, so we can leave this empty and rely on the default which calls + # `os.Hostname()`. + IdentifierInterfaceName: "{{ if eq .Values.mode \"statefulset\" }}{{ else }}eth0{{ end }}" RedisPeerManagement: # Host is the host and port of the Redis instance to use for peer cluster membership management.