Add support for memberlist bind network configuration

Signed-off-by: Ruben Vargas <[email protected]>
grafana · Oct 12, 2024 · 443116d · 443116d
1 parent 5238c4c
commit 443116d
Show file tree

Hide file tree

Showing 14 changed files with 175 additions and 7 deletions.
diff --git a/apis/tempo/v1alpha1/tempostack_types.go b/apis/tempo/v1alpha1/tempostack_types.go
@@ -303,6 +303,18 @@ var AllStatusConditions = []ConditionStatus{ConditionReady, ConditionFailed, Con
 // ConditionReason defines possible reasons for each condition.
 type ConditionReason string
 
+// InstanceAddrType defines the type of pod network to use for advertising IPs to the ring.
+//
+// +kubebuilder:validation:Enum=default;podIP
+type InstanceAddrType string
+
+const (
+	// InstanceAddrDefault when using the first from any private network interfaces (RFC 1918 and RFC 6598).
+	InstanceAddrDefault InstanceAddrType = "default"
+	// InstanceAddrPodIP when using the public pod IP from the cluster's pod network.
+	InstanceAddrPodIP InstanceAddrType = "podIP"
+)
+
 const (
 	// ReasonReady defines a healthy tempo instance.
 	ReasonReady ConditionReason = "Ready"
@@ -423,6 +435,16 @@ type MemberListSpec struct {
 	// +kubebuilder:validation:Optional
 	// +operator-sdk:csv:customresourcedefinitions:type=spec,xDescriptors="urn:alm:descriptor:com.tectonic.ui:booleanSwitch",displayName="Enable IPv6"
 	EnableIPv6 *bool `json:"enableIPv6,omitempty"`
+
+	// InstanceAddrType defines the type of address to use to advertise to the ring.
+	// Defaults to the first address from any private network interfaces of the current pod.
+	// Alternatively the public pod IP can be used in case private networks (RFC 1918 and RFC 6598)
+	// are not available.
+	//
+	// +optional
+	// +kubebuilder:validation:optional
+	// +operator-sdk:csv:customresourcedefinitions:type=spec,xDescriptors={"urn:alm:descriptor:com.tectonic.ui:select:default","urn:alm:descriptor:com.tectonic.ui:select:podIP"},displayName="Instance Address"
+	InstanceAddrType InstanceAddrType `json:"instanceAddrType,omitempty"`
 }
 
 // HashRingSpec defines the hash ring configuration.

diff --git a/bundle/community/manifests/tempo-operator.clusterserviceversion.yaml b/bundle/community/manifests/tempo-operator.clusterserviceversion.yaml
@@ -74,7 +74,7 @@ metadata:
     capabilities: Deep Insights
     categories: Logging & Tracing,Monitoring
     containerImage: ghcr.io/grafana/tempo-operator/tempo-operator:v0.13.0
-    createdAt: "2024-10-10T15:57:46Z"
+    createdAt: "2024-10-12T18:04:08Z"
     description: Create and manage deployments of Tempo, a high-scale distributed
       tracing backend.
     operatorframework.io/cluster-monitoring: "true"
@@ -584,6 +584,15 @@ spec:
         path: hashRing.memberlist.enableIPv6
         x-descriptors:
         - urn:alm:descriptor:com.tectonic.ui:booleanSwitch
+      - description: InstanceAddrType defines the type of address to use to advertise
+          to the ring. Defaults to the first address from any private network interfaces
+          of the current pod. Alternatively the public pod IP can be used in case
+          private networks (RFC 1918 and RFC 6598) are not available.
+        displayName: Instance Address
+        path: hashRing.memberlist.instanceAddrType
+        x-descriptors:
+        - urn:alm:descriptor:com.tectonic.ui:select:default
+        - urn:alm:descriptor:com.tectonic.ui:select:podIP
       - description: Images defines the image for each container.
         displayName: Container Images
         path: images

diff --git a/bundle/community/manifests/tempo.grafana.com_tempostacks.yaml b/bundle/community/manifests/tempo.grafana.com_tempostacks.yaml
@@ -79,6 +79,16 @@ spec:
                         description: EnableIPv6 enables IPv6 support for the memberlist
                           based hash ring.
                         type: boolean
+                      instanceAddrType:
+                        description: |-
+                          InstanceAddrType defines the type of address to use to advertise to the ring.
+                          Defaults to the first address from any private network interfaces of the current pod.
+                          Alternatively the public pod IP can be used in case private networks (RFC 1918 and RFC 6598)
+                          are not available.
+                        enum:
+                        - default
+                        - podIP
+                        type: string
                     type: object
                 type: object
               images:

diff --git a/bundle/openshift/manifests/tempo-operator.clusterserviceversion.yaml b/bundle/openshift/manifests/tempo-operator.clusterserviceversion.yaml
@@ -74,7 +74,7 @@ metadata:
     capabilities: Deep Insights
     categories: Logging & Tracing,Monitoring
     containerImage: ghcr.io/grafana/tempo-operator/tempo-operator:v0.13.0
-    createdAt: "2024-10-10T15:57:44Z"
+    createdAt: "2024-10-12T18:04:06Z"
     description: Create and manage deployments of Tempo, a high-scale distributed
       tracing backend.
     operatorframework.io/cluster-monitoring: "true"
@@ -584,6 +584,15 @@ spec:
         path: hashRing.memberlist.enableIPv6
         x-descriptors:
         - urn:alm:descriptor:com.tectonic.ui:booleanSwitch
+      - description: InstanceAddrType defines the type of address to use to advertise
+          to the ring. Defaults to the first address from any private network interfaces
+          of the current pod. Alternatively the public pod IP can be used in case
+          private networks (RFC 1918 and RFC 6598) are not available.
+        displayName: Instance Address
+        path: hashRing.memberlist.instanceAddrType
+        x-descriptors:
+        - urn:alm:descriptor:com.tectonic.ui:select:default
+        - urn:alm:descriptor:com.tectonic.ui:select:podIP
       - description: Images defines the image for each container.
         displayName: Container Images
         path: images

diff --git a/bundle/openshift/manifests/tempo.grafana.com_tempostacks.yaml b/bundle/openshift/manifests/tempo.grafana.com_tempostacks.yaml
@@ -79,6 +79,16 @@ spec:
                         description: EnableIPv6 enables IPv6 support for the memberlist
                           based hash ring.
                         type: boolean
+                      instanceAddrType:
+                        description: |-
+                          InstanceAddrType defines the type of address to use to advertise to the ring.
+                          Defaults to the first address from any private network interfaces of the current pod.
+                          Alternatively the public pod IP can be used in case private networks (RFC 1918 and RFC 6598)
+                          are not available.
+                        enum:
+                        - default
+                        - podIP
+                        type: string
                     type: object
                 type: object
               images:

diff --git a/config/crd/bases/tempo.grafana.com_tempostacks.yaml b/config/crd/bases/tempo.grafana.com_tempostacks.yaml
@@ -75,6 +75,16 @@ spec:
                         description: EnableIPv6 enables IPv6 support for the memberlist
                           based hash ring.
                         type: boolean
+                      instanceAddrType:
+                        description: |-
+                          InstanceAddrType defines the type of address to use to advertise to the ring.
+                          Defaults to the first address from any private network interfaces of the current pod.
+                          Alternatively the public pod IP can be used in case private networks (RFC 1918 and RFC 6598)
+                          are not available.
+                        enum:
+                        - default
+                        - podIP
+                        type: string
                     type: object
                 type: object
               images:

diff --git a/config/manifests/community/bases/tempo-operator.clusterserviceversion.yaml b/config/manifests/community/bases/tempo-operator.clusterserviceversion.yaml
@@ -513,6 +513,15 @@ spec:
         path: hashRing.memberlist.enableIPv6
         x-descriptors:
         - urn:alm:descriptor:com.tectonic.ui:booleanSwitch
+      - description: InstanceAddrType defines the type of address to use to advertise
+          to the ring. Defaults to the first address from any private network interfaces
+          of the current pod. Alternatively the public pod IP can be used in case
+          private networks (RFC 1918 and RFC 6598) are not available.
+        displayName: Instance Address
+        path: hashRing.memberlist.instanceAddrType
+        x-descriptors:
+        - urn:alm:descriptor:com.tectonic.ui:select:default
+        - urn:alm:descriptor:com.tectonic.ui:select:podIP
       - description: Images defines the image for each container.
         displayName: Container Images
         path: images

diff --git a/config/manifests/openshift/bases/tempo-operator.clusterserviceversion.yaml b/config/manifests/openshift/bases/tempo-operator.clusterserviceversion.yaml
@@ -513,6 +513,15 @@ spec:
         path: hashRing.memberlist.enableIPv6
         x-descriptors:
         - urn:alm:descriptor:com.tectonic.ui:booleanSwitch
+      - description: InstanceAddrType defines the type of address to use to advertise
+          to the ring. Defaults to the first address from any private network interfaces
+          of the current pod. Alternatively the public pod IP can be used in case
+          private networks (RFC 1918 and RFC 6598) are not available.
+        displayName: Instance Address
+        path: hashRing.memberlist.instanceAddrType
+        x-descriptors:
+        - urn:alm:descriptor:com.tectonic.ui:select:default
+        - urn:alm:descriptor:com.tectonic.ui:select:podIP
       - description: Images defines the image for each container.
         displayName: Container Images
         path: images

diff --git a/internal/manifests/config/build.go b/internal/manifests/config/build.go
@@ -4,6 +4,7 @@ import (
 	"bytes"
 	"embed"
 	"fmt"
+	"github.com/grafana/tempo-operator/internal/manifests/memberlist"
 	"html/template"
 	"io"
 	"path"
@@ -73,8 +74,9 @@ func buildConfiguration(params manifestutils.Params) ([]byte, error) {
 		StorageParams:   params.StorageParams,
 		GlobalRetention: tempo.Spec.Retention.Global.Traces.Duration.String(),
 		MemberList: memberlistOptions{
-			JoinMembers: []string{naming.Name("gossip-ring", tempo.Name)},
-			EnableIPv6:  ptr.Deref(tempo.Spec.HashRing.MemberList.EnableIPv6, false),
+			JoinMembers:  []string{naming.Name("gossip-ring", tempo.Name)},
+			EnableIPv6:   ptr.Deref(tempo.Spec.HashRing.MemberList.EnableIPv6, false),
+			InstanceAddr: gossipRingInstanceAddr(tempo.Spec.HashRing),
 		},
 		QueryFrontendDiscovery: fmt.Sprintf("%s:%d", naming.Name("query-frontend-discovery", tempo.Name), manifestutils.PortGRPCServer),
 		GlobalRateLimits:       fromRateLimitSpecToRateLimitOptions(tempo.Spec.LimitSpec.Global),
@@ -256,3 +258,17 @@ func renderTempoQueryTemplate(opts tempoQueryOptions) ([]byte, error) {
 
 	return cfg, nil
 }
+
+func gossipRingInstanceAddr(spec v1alpha1.HashRingSpec) string {
+	var instanceAddr string
+	switch spec.MemberList.InstanceAddrType {
+	case v1alpha1.InstanceAddrPodIP:
+		instanceAddr = fmt.Sprintf("${%s}", memberlist.GossipInstanceAddrEnvVarName)
+	case v1alpha1.InstanceAddrDefault:
+		// Do nothing use loki defaults
+	default:
+		// Do nothing use loki defaults
+	}
+
+	return instanceAddr
+}
diff --git a/internal/manifests/config/options.go b/internal/manifests/config/options.go
@@ -74,8 +74,9 @@ type tlsOptions struct {
 }
 
 type memberlistOptions struct {
-	JoinMembers []string
-	EnableIPv6  bool
+	JoinMembers  []string
+	EnableIPv6   bool
+	InstanceAddr string
 }
 
 type receiverTLSOptions struct {

diff --git a/internal/manifests/config/tempo-config.yaml b/internal/manifests/config/tempo-config.yaml
@@ -90,11 +90,17 @@ distributor:
   ring:
     kvstore:
       store: memberlist
+    {{ - with .MemberList.InstanceAddr }}
+    instance_addr: {{ . }}
+    {{ - end }}
 ingester:
   lifecycler:
     ring:
       kvstore:
         store: memberlist
+      {{ - with .MemberList.InstanceAddr }}
+      instance_addr: {{ . }}
+      {{ - end }}
       replication_factor: {{ .ReplicationFactor }}
     tokens_file_path: /var/tempo/tokens.json
     {{- if .MemberList.EnableIPv6 }}

diff --git a/internal/manifests/distributor/distributor.go b/internal/manifests/distributor/distributor.go
@@ -245,6 +245,7 @@ func deployment(params manifestutils.Params) *v1.Deployment {
 								"-target=distributor",
 								"-config.file=/conf/tempo.yaml",
 								"-log.level=info",
+								"-config.expand-env=true",
 							},
 							Ports:          containerPorts,
 							ReadinessProbe: manifestutils.TempoReadinessProbe(params.CtrlConfig.Gates.HTTPEncryption),

diff --git a/internal/manifests/ingester/ingester.go b/internal/manifests/ingester/ingester.go
@@ -94,6 +94,7 @@ func statefulSet(params manifestutils.Params) (*v1.StatefulSet, error) {
 								"-target=ingester",
 								"-config.file=/conf/tempo.yaml",
 								"-log.level=info",
+								"-config.expand-env=true",
 							},
 							VolumeMounts: []corev1.VolumeMount{
 								{

diff --git a/internal/manifests/memberlist/gossip.go b/internal/manifests/memberlist/gossip.go
@@ -1,18 +1,21 @@
 package memberlist
 
 import (
+	"github.com/imdario/mergo"
 	corev1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	k8slabels "k8s.io/apimachinery/pkg/labels"
 	"k8s.io/apimachinery/pkg/util/intstr"
+	"strings"
 
 	"github.com/grafana/tempo-operator/apis/tempo/v1alpha1"
 	"github.com/grafana/tempo-operator/internal/manifests/manifestutils"
 	"github.com/grafana/tempo-operator/internal/manifests/naming"
 )
 
 const (
-	componentName = "gossip-ring"
+	componentName                = "gossip-ring"
+	GossipInstanceAddrEnvVarName = "HASH_RING_INSTANCE_ADDR"
 )
 
 var (
@@ -46,3 +49,55 @@ func BuildGossip(tempo v1alpha1.TempoStack) *corev1.Service {
 		},
 	}
 }
+
+func resetEnvVar(podSpec *corev1.PodSpec, name string) {
+	for i, container := range podSpec.Containers {
+		found, index := findEnvVar(name, container.Env)
+		if found {
+			podSpec.Containers[i].Env = append(podSpec.Containers[i].Env[:index], podSpec.Containers[i].Env[index+1:]...)
+		}
+	}
+}
+
+func findEnvVar(name string, envVars []corev1.EnvVar) (bool, int) {
+	for i, env := range envVars {
+		if env.Name == name || env.Name == strings.ToLower(name) {
+			return true, i
+		}
+	}
+	return false, 0
+}
+
+func configureHashRingEnv(p *corev1.PodSpec, tempo v1alpha1.TempoStack) error {
+	resetEnvVar(p, GossipInstanceAddrEnvVarName)
+
+	memberList := tempo.Spec.HashRing.MemberList
+	enableIPV6 := memberList.EnableIPv6 != nil && *memberList.EnableIPv6
+
+	if !enableIPV6 && memberList.InstanceAddrType != v1alpha1.InstanceAddrPodIP {
+		return nil
+	}
+
+	src := corev1.Container{
+		Env: []corev1.EnvVar{
+			{
+				Name: GossipInstanceAddrEnvVarName,
+				ValueFrom: &corev1.EnvVarSource{
+					FieldRef: &corev1.ObjectFieldSelector{
+						APIVersion: "v1",
+						FieldPath:  "status.podIP",
+					},
+				},
+			},
+		},
+	}
+
+	for i, dst := range p.Containers {
+		if err := mergo.Merge(&dst, src, mergo.WithAppendSlice); err != nil {
+			return err
+		}
+		p.Containers[i] = dst
+	}
+
+	return nil
+}