Add support for memberlist bind network configuration

.chloggen/fix_hashring.yaml

@@ -0,0 +1,19 @@
+# One of 'breaking', 'deprecation', 'new_component', 'enhancement', 'bug_fix'
+change_type: bug_fix
+
+# The name of the component, or a single word describing the area of concern, (e.g. tempostack, tempomonolithic, github action)
+component: tempostack
+
+# A brief description of the change. Surround your text with quotes ("") if it needs to start with a backtick (`).
+note: Add support for memberlist bind network configuration
+
+# One or more tracking issues related to the change
+issues: [1060]
+
+# (Optional) One or more lines of additional information to render under the primary note.
+# These lines will be padded with 2 spaces and then inserted directly into the document.
+# Use pipe (|) for multiline entries.
+subtext: |
+  Adds support to configure the memberlist bind_addr field using the pod network IP range instead of the default private network range used. 
+  In managed Kubernetes/OpenShift cluster environments as well as in special on-prem setup the private IP range might not be available for using them. 
+  With this change set the TempoStack administrator can choose as a bind address the current pod network IP assigned by the cluster's pod network.

apis/tempo/v1alpha1/tempostack_types.go

@@ -303,6 +303,18 @@ var AllStatusConditions = []ConditionStatus{ConditionReady, ConditionFailed, Con
 // ConditionReason defines possible reasons for each condition.
 type ConditionReason string
 
+// InstanceAddrType defines the type of pod network to use for advertising IPs to the ring.
+//
+// +kubebuilder:validation:Enum=default;podIP
+type InstanceAddrType string
+
+const (
+	// InstanceAddrDefault when using the first from any private network interfaces (RFC 1918 and RFC 6598).
+	InstanceAddrDefault InstanceAddrType = "default"
+	// InstanceAddrPodIP when using the public pod IP from the cluster's pod network.
+	InstanceAddrPodIP InstanceAddrType = "podIP"
+)
+
 const (
 	// ReasonReady defines a healthy tempo instance.
 	ReasonReady ConditionReason = "Ready"
@@ -423,6 +435,16 @@ type MemberListSpec struct {
 	// +kubebuilder:validation:Optional
 	// +operator-sdk:csv:customresourcedefinitions:type=spec,xDescriptors="urn:alm:descriptor:com.tectonic.ui:booleanSwitch",displayName="Enable IPv6"
 	EnableIPv6 *bool `json:"enableIPv6,omitempty"`
+
+	// InstanceAddrType defines the type of address to use to advertise to the ring.
+	// Defaults to the first address from any private network interfaces of the current pod.
+	// Alternatively the public pod IP can be used in case private networks (RFC 1918 and RFC 6598)
+	// are not available.
+	//
+	// +optional
+	// +kubebuilder:validation:optional
+	// +operator-sdk:csv:customresourcedefinitions:type=spec,xDescriptors={"urn:alm:descriptor:com.tectonic.ui:select:default","urn:alm:descriptor:com.tectonic.ui:select:podIP"},displayName="Instance Address"
+	InstanceAddrType InstanceAddrType `json:"instanceAddrType,omitempty"`
 }
 
 // HashRingSpec defines the hash ring configuration.

bundle/community/manifests/tempo-operator.clusterserviceversion.yaml

@@ -74,7 +74,7 @@ metadata:
     capabilities: Deep Insights
     categories: Logging & Tracing,Monitoring
     containerImage: ghcr.io/grafana/tempo-operator/tempo-operator:v0.13.0
-    createdAt: "2024-10-10T15:57:46Z"
+    createdAt: "2024-10-12T18:04:08Z"
     description: Create and manage deployments of Tempo, a high-scale distributed
       tracing backend.
     operatorframework.io/cluster-monitoring: "true"
@@ -584,6 +584,15 @@ spec:
         path: hashRing.memberlist.enableIPv6
         x-descriptors:
         - urn:alm:descriptor:com.tectonic.ui:booleanSwitch
+      - description: InstanceAddrType defines the type of address to use to advertise
+          to the ring. Defaults to the first address from any private network interfaces
+          of the current pod. Alternatively the public pod IP can be used in case
+          private networks (RFC 1918 and RFC 6598) are not available.
+        displayName: Instance Address
+        path: hashRing.memberlist.instanceAddrType
+        x-descriptors:
+        - urn:alm:descriptor:com.tectonic.ui:select:default
+        - urn:alm:descriptor:com.tectonic.ui:select:podIP
       - description: Images defines the image for each container.
         displayName: Container Images
         path: images

bundle/community/manifests/tempo.grafana.com_tempostacks.yaml

@@ -79,6 +79,16 @@ spec:
                         description: EnableIPv6 enables IPv6 support for the memberlist
                           based hash ring.
                         type: boolean
+                      instanceAddrType:
+                        description: |-
+                          InstanceAddrType defines the type of address to use to advertise to the ring.
+                          Defaults to the first address from any private network interfaces of the current pod.
+                          Alternatively the public pod IP can be used in case private networks (RFC 1918 and RFC 6598)
+                          are not available.
+                        enum:
+                        - default
+                        - podIP
+                        type: string
                     type: object
                 type: object
               images:

bundle/openshift/manifests/tempo-operator.clusterserviceversion.yaml

@@ -74,7 +74,7 @@ metadata:
     capabilities: Deep Insights
     categories: Logging & Tracing,Monitoring
     containerImage: ghcr.io/grafana/tempo-operator/tempo-operator:v0.13.0
-    createdAt: "2024-10-10T15:57:44Z"
+    createdAt: "2024-10-12T18:04:06Z"
     description: Create and manage deployments of Tempo, a high-scale distributed
       tracing backend.
     operatorframework.io/cluster-monitoring: "true"
@@ -584,6 +584,15 @@ spec:
         path: hashRing.memberlist.enableIPv6
         x-descriptors:
         - urn:alm:descriptor:com.tectonic.ui:booleanSwitch
+      - description: InstanceAddrType defines the type of address to use to advertise
+          to the ring. Defaults to the first address from any private network interfaces
+          of the current pod. Alternatively the public pod IP can be used in case
+          private networks (RFC 1918 and RFC 6598) are not available.
+        displayName: Instance Address
+        path: hashRing.memberlist.instanceAddrType
+        x-descriptors:
+        - urn:alm:descriptor:com.tectonic.ui:select:default
+        - urn:alm:descriptor:com.tectonic.ui:select:podIP
       - description: Images defines the image for each container.
         displayName: Container Images
         path: images

bundle/openshift/manifests/tempo.grafana.com_tempostacks.yaml

@@ -79,6 +79,16 @@ spec:
                         description: EnableIPv6 enables IPv6 support for the memberlist
                           based hash ring.
                         type: boolean
+                      instanceAddrType:
+                        description: |-
+                          InstanceAddrType defines the type of address to use to advertise to the ring.
+                          Defaults to the first address from any private network interfaces of the current pod.
+                          Alternatively the public pod IP can be used in case private networks (RFC 1918 and RFC 6598)
+                          are not available.
+                        enum:
+                        - default
+                        - podIP
+                        type: string
                     type: object
                 type: object
               images:

config/crd/bases/tempo.grafana.com_tempostacks.yaml

@@ -75,6 +75,16 @@ spec:
                         description: EnableIPv6 enables IPv6 support for the memberlist
                           based hash ring.
                         type: boolean
+                      instanceAddrType:
+                        description: |-
+                          InstanceAddrType defines the type of address to use to advertise to the ring.
+                          Defaults to the first address from any private network interfaces of the current pod.
+                          Alternatively the public pod IP can be used in case private networks (RFC 1918 and RFC 6598)
+                          are not available.
+                        enum:
+                        - default
+                        - podIP
+                        type: string
                     type: object
                 type: object
               images:

config/manifests/community/bases/tempo-operator.clusterserviceversion.yaml

@@ -513,6 +513,15 @@ spec:
         path: hashRing.memberlist.enableIPv6
         x-descriptors:
         - urn:alm:descriptor:com.tectonic.ui:booleanSwitch
+      - description: InstanceAddrType defines the type of address to use to advertise
+          to the ring. Defaults to the first address from any private network interfaces
+          of the current pod. Alternatively the public pod IP can be used in case
+          private networks (RFC 1918 and RFC 6598) are not available.
+        displayName: Instance Address
+        path: hashRing.memberlist.instanceAddrType
+        x-descriptors:
+        - urn:alm:descriptor:com.tectonic.ui:select:default
+        - urn:alm:descriptor:com.tectonic.ui:select:podIP
       - description: Images defines the image for each container.
         displayName: Container Images
         path: images

config/manifests/openshift/bases/tempo-operator.clusterserviceversion.yaml

@@ -513,6 +513,15 @@ spec:
         path: hashRing.memberlist.enableIPv6
         x-descriptors:
         - urn:alm:descriptor:com.tectonic.ui:booleanSwitch
+      - description: InstanceAddrType defines the type of address to use to advertise
+          to the ring. Defaults to the first address from any private network interfaces
+          of the current pod. Alternatively the public pod IP can be used in case
+          private networks (RFC 1918 and RFC 6598) are not available.
+        displayName: Instance Address
+        path: hashRing.memberlist.instanceAddrType
+        x-descriptors:
+        - urn:alm:descriptor:com.tectonic.ui:select:default
+        - urn:alm:descriptor:com.tectonic.ui:select:podIP
       - description: Images defines the image for each container.
         displayName: Container Images
         path: images

docs/spec/tempo.grafana.com_tempostacks.yaml

@@ -8,6 +8,7 @@ spec:                                    # TempoStackSpec defines the desired st
   hashRing:                              # HashRing defines the spec for the distributed hash ring configuration.
     memberlist:                          # MemberList configuration spec
       enableIPv6: false                  # EnableIPv6 enables IPv6 support for the memberlist based hash ring.
+      instanceAddrType: ""               # InstanceAddrType defines the type of address to use to advertise to the ring. Defaults to the first address from any private network interfaces of the current pod. Alternatively the public pod IP can be used in case private networks (RFC 1918 and RFC 6598) are not available.
   images:                                # Images defines the image for each container.
     jaegerQuery: ""                      # JaegerQuery defines the tempo-query container image.
     oauthProxy: ""                       # OauthProxy defines the oauth proxy image used to protect the jaegerUI on single tenant.

internal/manifests/compactor/compactor.go

@@ -27,6 +27,11 @@ func BuildCompactor(params manifestutils.Params) ([]client.Object, error) {
 	}
 	gates := params.CtrlConfig.Gates
 	tempo := params.Tempo
+
+	if err := memberlist.ConfigureHashRingEnv(&d.Spec.Template.Spec, params.Tempo); err != nil {
+		return nil, err
+	}
+
 	if gates.HTTPEncryption || gates.GRPCEncryption {
 		caBundleName := naming.SigningCABundleName(tempo.Name)
 		if err := manifestutils.ConfigureServiceCA(&d.Spec.Template.Spec, caBundleName); err != nil {
@@ -90,6 +95,7 @@ func deployment(params manifestutils.Params) (*v1.Deployment, error) {
 								"-target=compactor",
 								"-config.file=/conf/tempo.yaml",
 								"-log.level=info",
+								"-config.expand-env=true",
 							},
 							Ports: []corev1.ContainerPort{
 								{

internal/manifests/compactor/compactor_test.go

@@ -117,6 +117,7 @@ func TestBuildCompactor(t *testing.T) {
 								"-target=compactor",
 								"-config.file=/conf/tempo.yaml",
 								"-log.level=info",
+								"-config.expand-env=true",
 							},
 							VolumeMounts: []corev1.VolumeMount{
 								{

internal/manifests/config/build.go

@@ -12,6 +12,7 @@ import (
 
 	"github.com/grafana/tempo-operator/apis/tempo/v1alpha1"
 	"github.com/grafana/tempo-operator/internal/manifests/manifestutils"
+	"github.com/grafana/tempo-operator/internal/manifests/memberlist"
 	"github.com/grafana/tempo-operator/internal/manifests/naming"
 )
 
@@ -73,8 +74,9 @@ func buildConfiguration(params manifestutils.Params) ([]byte, error) {
 		StorageParams:   params.StorageParams,
 		GlobalRetention: tempo.Spec.Retention.Global.Traces.Duration.String(),
 		MemberList: memberlistOptions{
-			JoinMembers: []string{naming.Name("gossip-ring", tempo.Name)},
-			EnableIPv6:  ptr.Deref(tempo.Spec.HashRing.MemberList.EnableIPv6, false),
+			JoinMembers:  []string{naming.Name("gossip-ring", tempo.Name)},
+			EnableIPv6:   ptr.Deref(tempo.Spec.HashRing.MemberList.EnableIPv6, false),
+			InstanceAddr: gossipRingInstanceAddr(tempo.Spec.HashRing),
 		},
 		QueryFrontendDiscovery: fmt.Sprintf("%s:%d", naming.Name("query-frontend-discovery", tempo.Name), manifestutils.PortGRPCServer),
 		GlobalRateLimits:       fromRateLimitSpecToRateLimitOptions(tempo.Spec.LimitSpec.Global),
@@ -256,3 +258,17 @@ func renderTempoQueryTemplate(opts tempoQueryOptions) ([]byte, error) {
 
 	return cfg, nil
 }
+
+func gossipRingInstanceAddr(spec v1alpha1.HashRingSpec) string {
+	var instanceAddr string
+	switch spec.MemberList.InstanceAddrType {
+	case v1alpha1.InstanceAddrPodIP:
+		instanceAddr = fmt.Sprintf("${%s}", memberlist.GossipInstanceAddrEnvVarName)
+	case v1alpha1.InstanceAddrDefault:
+		// Do nothing use loki defaults
+	default:
+		// Do nothing use loki defaults
+	}
+
+	return instanceAddr
+}

internal/manifests/config/options.go

@@ -74,8 +74,9 @@ type tlsOptions struct {
 }
 
 type memberlistOptions struct {
-	JoinMembers []string
-	EnableIPv6  bool
+	JoinMembers  []string
+	EnableIPv6   bool
+	InstanceAddr string
 }
 
 type receiverTLSOptions struct {

internal/manifests/config/tempo-config.yaml

@@ -90,11 +90,17 @@ distributor:
   ring:
     kvstore:
       store: memberlist
+    {{- with .MemberList.InstanceAddr }}
+    instance_addr: {{ . }}
+    {{- end }}
 ingester:
   lifecycler:
     ring:
       kvstore:
         store: memberlist
+      {{- with .MemberList.InstanceAddr }}
+      instance_addr: {{ . }}
+      {{- end }}
       replication_factor: {{ .ReplicationFactor }}
     tokens_file_path: /var/tempo/tokens.json
     {{- if .MemberList.EnableIPv6 }}

internal/manifests/distributor/distributor.go

@@ -24,6 +24,11 @@ func BuildDistributor(params manifestutils.Params) ([]client.Object, error) {
 	dep := deployment(params)
 	var err error
 	dep.Spec.Template, err = manifestutils.PatchTracingJaegerEnv(params.Tempo, dep.Spec.Template)
+
+	if err := memberlist.ConfigureHashRingEnv(&dep.Spec.Template.Spec, params.Tempo); err != nil {
+		return nil, err
+	}
+
 	if err != nil {
 		return nil, err
 	}
@@ -245,6 +250,7 @@ func deployment(params manifestutils.Params) *v1.Deployment {
 								"-target=distributor",
 								"-config.file=/conf/tempo.yaml",
 								"-log.level=info",
+								"-config.expand-env=true",
 							},
 							Ports:          containerPorts,
 							ReadinessProbe: manifestutils.TempoReadinessProbe(params.CtrlConfig.Gates.HTTPEncryption),

internal/manifests/distributor/distributor_test.go

@@ -879,6 +879,7 @@ func TestBuildDistributor(t *testing.T) {
 										"-target=distributor",
 										"-config.file=/conf/tempo.yaml",
 										"-log.level=info",
+										"-config.expand-env=true",
 									},
 									Env:          []corev1.EnvVar{},
 									VolumeMounts: ts.expectedVolumeMounts,

internal/manifests/ingester/ingester.go

@@ -27,6 +27,10 @@ func BuildIngester(params manifestutils.Params) ([]client.Object, error) {
 		return nil, err
 	}
 
+	if err := memberlist.ConfigureHashRingEnv(&ss.Spec.Template.Spec, params.Tempo); err != nil {
+		return nil, err
+	}
+
 	gates := params.CtrlConfig.Gates
 	tempo := params.Tempo
 
@@ -94,6 +98,7 @@ func statefulSet(params manifestutils.Params) (*v1.StatefulSet, error) {
 								"-target=ingester",
 								"-config.file=/conf/tempo.yaml",
 								"-log.level=info",
+								"-config.expand-env=true",
 							},
 							VolumeMounts: []corev1.VolumeMount{
 								{

internal/manifests/ingester/ingester_test.go

@@ -96,6 +96,7 @@ func TestBuildIngester(t *testing.T) {
 							Args: []string{"-target=ingester",
 								"-config.file=/conf/tempo.yaml",
 								"-log.level=info",
+								"-config.expand-env=true",
 								"--storage.trace.s3.secret_key=$(S3_SECRET_KEY)",
 								"--storage.trace.s3.access_key=$(S3_ACCESS_KEY)",
 							},