Merge pull request #613 from Vincent056/metric_token

OCPBUGS-39417: Add service account and token for service monitoring

bundle/manifests/compliance-operator-metrics_rbac.authorization.k8s.io_v1_rolebinding.yaml

@@ -8,6 +8,9 @@ roleRef:
   kind: ClusterRole
   name: compliance-operator-metrics
 subjects:
+- kind: ServiceAccount
+  name: compliance-operator-metrics
+  namespace: openshift-compliance
 - kind: ServiceAccount
   name: prometheus-k8s
   namespace: openshift-monitoring

bundle/manifests/compliance-operator-metrics_v1_serviceaccount.yaml

@@ -0,0 +1,5 @@
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  creationTimestamp: null
+  name: compliance-operator-metrics

cmd/manager/common.go

@@ -26,8 +26,10 @@ import (
 )
 
 const (
-	maxRetries             = 15
-	maxRetriesForTimestamp = 3
+	maxRetries                          = 15
+	maxRetriesForTimestamp              = 3
+	complianceOperatorMetricsSA         = "compliance-operator-metrics"
+	complianceOperatorMetricsSecretName = "compliance-operator-metrics-token"
 )
 
 var cmdLog = logf.Log.WithName("cmd")

cmd/manager/operator.go

@@ -439,7 +439,7 @@ func addMetrics(ctx context.Context, cfg *rest.Config, kClient *kubernetes.Clien
 		os.Exit(1)
 	}
 
-	if err := handleServiceMonitor(ctx, cfg, mClient, kClient, operatorNs, metricsService); err != nil {
+	if err := handleServiceMonitor(ctx, cfg, mClient, operatorNs, metricsService); err != nil {
 		log.Error(err, "Error creating ServiceMonitor")
 		os.Exit(1)
 	}
@@ -531,6 +531,28 @@ func ensureMetricsServiceAndSecret(ctx context.Context, kClient *kubernetes.Clie
 		}
 	}
 
+	// Check if the metrics service account token secret exists. If not, create it and trigger a restart.
+	_, err = kClient.CoreV1().Secrets(ns).Get(ctx, complianceOperatorMetricsSecretName, metav1.GetOptions{})
+	if err != nil {
+		if kerr.IsNotFound(err) {
+			secret := &v1.Secret{
+				ObjectMeta: metav1.ObjectMeta{
+					Name:      complianceOperatorMetricsSecretName,
+					Namespace: ns,
+					Annotations: map[string]string{
+						"kubernetes.io/service-account.name": complianceOperatorMetricsSA,
+					},
+				},
+				Type: v1.SecretTypeServiceAccountToken,
+			}
+			if _, createErr := kClient.CoreV1().Secrets(ns).Create(context.TODO(), secret, metav1.CreateOptions{}); createErr != nil && !kerr.IsAlreadyExists(createErr) {
+				return nil, createErr
+			}
+			return nil, errors.New("operator metrics token not found; restarting as the service may have just been created")
+		}
+		return nil, err
+	}
+
 	return returnService, nil
 }
 
@@ -681,7 +703,7 @@ func getDefaultRoles(platform PlatformType) []string {
 	return defaultRolesPerPlatform[PlatformGeneric]
 }
 
-func generateOperatorServiceMonitor(service *v1.Service, namespace, secretName string) *monitoring.ServiceMonitor {
+func generateOperatorServiceMonitor(service *v1.Service, namespace string) *monitoring.ServiceMonitor {
 	serviceMonitor := GenerateServiceMonitor(service)
 	for i := range serviceMonitor.Spec.Endpoints {
 		if serviceMonitor.Spec.Endpoints[i].Port == ctrlMetrics.ControllerMetricsServiceName {
@@ -691,7 +713,7 @@ func generateOperatorServiceMonitor(service *v1.Service, namespace, secretName s
 				Type: "Bearer",
 				Credentials: &v1.SecretKeySelector{
 					LocalObjectReference: v1.LocalObjectReference{
-						Name: secretName,
+						Name: complianceOperatorMetricsSecretName,
 					},
 					Key: "token",
 				},
@@ -707,25 +729,6 @@ func generateOperatorServiceMonitor(service *v1.Service, namespace, secretName s
 	return serviceMonitor
 }
 
-func getSecretNameForServiceAccount(clientset *kubernetes.Clientset, namespace string, serviceAccountName string) (string, error) {
-	// List all secrets in the specified namespace
-	secrets, err := clientset.CoreV1().Secrets(namespace).List(context.TODO(), metav1.ListOptions{})
-	if err != nil {
-		return "", err
-	}
-
-	// Iterate through the secrets to find the one associated with the service account
-	for _, secret := range secrets.Items {
-		if secret.Annotations != nil {
-			if saName, exists := secret.Annotations["kubernetes.io/service-account.name"]; exists && saName == serviceAccountName {
-				return secret.Name, nil
-			}
-		}
-	}
-
-	return "", errors.New("secret for service account not found")
-}
-
 // createOrUpdateServiceMonitor creates or updates the ServiceMonitor if it already exists.
 func createOrUpdateServiceMonitor(ctx context.Context, mClient *monclientv1.MonitoringV1Client,
 	namespace string, serviceMonitor *monitoring.ServiceMonitor) error {
@@ -751,7 +754,7 @@ func createOrUpdateServiceMonitor(ctx context.Context, mClient *monclientv1.Moni
 
 // handleServiceMonitor attempts to create a ServiceMonitor out of service, and updates it to include the controller
 // metrics paths.
-func handleServiceMonitor(ctx context.Context, cfg *rest.Config, mClient *monclientv1.MonitoringV1Client, kubeClient *kubernetes.Clientset,
+func handleServiceMonitor(ctx context.Context, cfg *rest.Config, mClient *monclientv1.MonitoringV1Client,
 	namespace string, service *v1.Service) error {
 	ok, err := ResourceExists(discovery.NewDiscoveryClientForConfigOrDie(cfg),
 		"monitoring.coreos.com/v1", "ServiceMonitor")
@@ -763,16 +766,7 @@ func handleServiceMonitor(ctx context.Context, cfg *rest.Config, mClient *moncli
 		return nil
 	}
 
-	serviceAccountName := "compliance-operator"
-	secretName, err := getSecretNameForServiceAccount(kubeClient, namespace, serviceAccountName)
-	if err != nil {
-		if kerr.IsNotFound(err) {
-			log.Infof("Unable to find secret associated with %s service account: %s", serviceAccountName, err)
-		} else {
-			log.Errorf("Failed to retrieve secret associated with %s service account for setting up metrics monitor: %s", serviceAccountName, err)
-		}
-	}
-	serviceMonitor := generateOperatorServiceMonitor(service, namespace, secretName)
+	serviceMonitor := generateOperatorServiceMonitor(service, namespace)
 
 	return createOrUpdateServiceMonitor(ctx, mClient, namespace, serviceMonitor)
 }

cmd/manager/operator_test.go

@@ -24,7 +24,7 @@ var _ = Describe("Operator Startup Function tests", func() {
 		When("Installing to non-controlled namespace", func() {
 			It("ServiceMonitor is generated with the proper TLSConfig ServerName", func() {
 				metricService := operatorMetricService("foobar")
-				sm := generateOperatorServiceMonitor(metricService, "foobar", "secret")
+				sm := generateOperatorServiceMonitor(metricService, "foobar")
 				controllerMetricServiceFound := false
 				for _, ep := range sm.Spec.Endpoints {
 					if ep.Port == metrics.ControllerMetricsServiceName && ep.TLSConfig != nil {

config/rbac/kustomization.yaml

@@ -1,49 +1,50 @@
 resources:
-# All RBAC will be applied under this service account in
-# the deployment namespace. You may comment out this resource
-# if your manager will use a service account that exists at
-# runtime. Be sure to update RoleBinding and ClusterRoleBinding
-# subjects if changing service account names.
-- operator_service_account.yaml
-- operator_cluster_role.yaml
-- operator_cluster_role_binding.yaml
-- operator_role.yaml
-- operator_role_binding.yaml
-- api_resource_collector_service_account.yaml
-- api_resource_collector_role.yaml
-- api_resource_collector_role_binding.yaml
-- api_resource_collector_cluster_role.yaml
-- api_resource_collector_cluster_role_binding.yaml
-- profileparser_service_account.yaml
-- profileparser_role.yaml
-- profileparser_role_binding.yaml
-- remediation_aggregator_service_account.yaml
-- remediation_aggregator_role.yaml
-- remediation_aggregator_role_binding.yaml
-- remediation_aggregator_cluster_role.yaml
-- remediation_aggregator_cluster_role_binding.yaml
-- rerunner_service_account.yaml
-- rerunner_role.yaml
-- rerunner_role_binding.yaml
-- resultscollector_service_account.yaml
-- resultscollector_role.yaml
-- resultscollector_role_binding.yaml
-- resultserver_service_account.yaml
-- resultserver_role.yaml
-- resultserver_role_binding.yaml
-- leader_election_role.yaml
-- leader_election_role_binding.yaml
-- complianceremediation_editor_role.yaml
-- complianceremediation_viewer_role.yaml
-- compliancescan_editor_role.yaml
-- compliancescan_viewer_role.yaml
-- compliancesuite_editor_role.yaml
-- compliancesuite_viewer_role.yaml
-- profilebundle_editor_role.yaml
-- profilebundle_viewer_role.yaml
-- scansettingbinding_editor_role.yaml
-- scansettingbinding_viewer_role.yaml
-- tailoredprofile_editor_role.yaml
-- tailoredprofile_viewer_role.yaml
-- metrics_cluster_role.yaml
-- metrics_role_binding.yaml
+  # All RBAC will be applied under this service account in
+  # the deployment namespace. You may comment out this resource
+  # if your manager will use a service account that exists at
+  # runtime. Be sure to update RoleBinding and ClusterRoleBinding
+  # subjects if changing service account names.
+  - operator_service_account.yaml
+  - operator_cluster_role.yaml
+  - operator_cluster_role_binding.yaml
+  - operator_role.yaml
+  - operator_role_binding.yaml
+  - api_resource_collector_service_account.yaml
+  - api_resource_collector_role.yaml
+  - api_resource_collector_role_binding.yaml
+  - api_resource_collector_cluster_role.yaml
+  - api_resource_collector_cluster_role_binding.yaml
+  - profileparser_service_account.yaml
+  - profileparser_role.yaml
+  - profileparser_role_binding.yaml
+  - remediation_aggregator_service_account.yaml
+  - remediation_aggregator_role.yaml
+  - remediation_aggregator_role_binding.yaml
+  - remediation_aggregator_cluster_role.yaml
+  - remediation_aggregator_cluster_role_binding.yaml
+  - rerunner_service_account.yaml
+  - rerunner_role.yaml
+  - rerunner_role_binding.yaml
+  - resultscollector_service_account.yaml
+  - resultscollector_role.yaml
+  - resultscollector_role_binding.yaml
+  - resultserver_service_account.yaml
+  - resultserver_role.yaml
+  - resultserver_role_binding.yaml
+  - leader_election_role.yaml
+  - leader_election_role_binding.yaml
+  - complianceremediation_editor_role.yaml
+  - complianceremediation_viewer_role.yaml
+  - compliancescan_editor_role.yaml
+  - compliancescan_viewer_role.yaml
+  - compliancesuite_editor_role.yaml
+  - compliancesuite_viewer_role.yaml
+  - profilebundle_editor_role.yaml
+  - profilebundle_viewer_role.yaml
+  - scansettingbinding_editor_role.yaml
+  - scansettingbinding_viewer_role.yaml
+  - tailoredprofile_editor_role.yaml
+  - tailoredprofile_viewer_role.yaml
+  - metrics_cluster_role.yaml
+  - metrics_role_binding.yaml
+  - metrics_service_account.yaml

config/rbac/metrics_role_binding.yaml

@@ -8,6 +8,9 @@ roleRef:
   kind: ClusterRole
   name: compliance-operator-metrics
 subjects:
+  - kind: ServiceAccount
+    name: compliance-operator-metrics
+    namespace: openshift-compliance
   - kind: ServiceAccount
     name: prometheus-k8s
-    namespace: openshift-monitoring
+    namespace: openshift-monitoring

config/rbac/metrics_service_account.yaml

@@ -0,0 +1,4 @@
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: compliance-operator-metrics

go.mod

@@ -13,6 +13,55 @@ require (
 	sigs.k8s.io/controller-runtime v0.19.0
 )
 
+require (
+	github.com/Azure/azure-sdk-for-go/sdk/azcore v1.13.0 // indirect
+	github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.7.0 // indirect
+	github.com/Azure/azure-sdk-for-go/sdk/internal v1.10.0 // indirect
+	github.com/AzureAD/microsoft-authentication-library-for-go v1.2.2 // indirect
+	github.com/alecthomas/units v0.0.0-20240626203959-61d1e3462e30 // indirect
+	github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2 // indirect
+	github.com/bboreham/go-loser v0.0.0-20230920113527-fcc2c21820a3 // indirect
+	github.com/dennwc/varint v1.0.0 // indirect
+	github.com/edsrzf/mmap-go v1.1.0 // indirect
+	github.com/facette/natsort v0.0.0-20181210072756-2cd4dd1e2dcb // indirect
+	github.com/felixge/httpsnoop v1.0.4 // indirect
+	github.com/go-kit/log v0.2.1 // indirect
+	github.com/go-logfmt/logfmt v0.6.0 // indirect
+	github.com/go-logr/stdr v1.2.2 // indirect
+	github.com/go-openapi/analysis v0.22.2 // indirect
+	github.com/go-openapi/errors v0.22.0 // indirect
+	github.com/go-openapi/loads v0.21.5 // indirect
+	github.com/go-openapi/spec v0.20.14 // indirect
+	github.com/go-openapi/strfmt v0.23.0 // indirect
+	github.com/go-openapi/validate v0.23.0 // indirect
+	github.com/golang-jwt/jwt/v5 v5.2.1 // indirect
+	github.com/golang/snappy v0.0.4 // indirect
+	github.com/grafana/regexp v0.0.0-20240518133315-a468a5bfb3bc // indirect
+	github.com/jmespath/go-jmespath v0.4.0 // indirect
+	github.com/jpillora/backoff v1.0.0 // indirect
+	github.com/julienschmidt/httprouter v1.3.0 // indirect
+	github.com/klauspost/compress v1.17.9 // indirect
+	github.com/kylelemons/godebug v1.1.0 // indirect
+	github.com/mitchellh/mapstructure v1.5.0 // indirect
+	github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f // indirect
+	github.com/oklog/ulid v1.3.1 // indirect
+	github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c // indirect
+	github.com/prometheus/alertmanager v0.27.0 // indirect
+	github.com/prometheus/common/sigv4 v0.1.0 // indirect
+	go.mongodb.org/mongo-driver v1.14.0 // indirect
+	go.opentelemetry.io/collector/pdata v1.12.0 // indirect
+	go.opentelemetry.io/collector/semconv v0.105.0 // indirect
+	go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.53.0 // indirect
+	go.opentelemetry.io/otel v1.28.0 // indirect
+	go.opentelemetry.io/otel/metric v1.28.0 // indirect
+	go.opentelemetry.io/otel/trace v1.28.0 // indirect
+	go.uber.org/atomic v1.11.0 // indirect
+	go.uber.org/goleak v1.3.0 // indirect
+	golang.org/x/crypto v0.26.0 // indirect
+	google.golang.org/genproto/googleapis/rpc v0.0.0-20240708141625-4ad9e859172b // indirect
+	google.golang.org/grpc v1.65.0 // indirect
+)
+
 require (
 	github.com/ajeddeloh/go-json v0.0.0-20200220154158-5ae607161559 // indirect
 	github.com/antchfx/xmlquery v1.4.1
@@ -34,7 +83,7 @@ require (
 	github.com/google/go-cmp v0.6.0
 	github.com/google/gofuzz v1.2.0 // indirect
 	github.com/google/uuid v1.6.0
-	github.com/imdario/mergo v0.3.13 // indirect
+	github.com/imdario/mergo v0.3.16 // indirect
 	github.com/itchyny/gojq v0.12.16
 	github.com/jaytaylor/html2text v0.0.0-20230321000545-74c2419ad056
 	github.com/json-iterator/go v1.1.12 // indirect
@@ -90,7 +139,7 @@ require (
 
 require (
 	github.com/antchfx/xpath v1.3.1 // indirect
-	github.com/aws/aws-sdk-go v1.53.5 // indirect
+	github.com/aws/aws-sdk-go v1.54.19 // indirect
 	github.com/ccojocar/zxcvbn-go v1.0.2 // indirect
 	github.com/coreos/fcct v0.5.0 // indirect
 	github.com/coreos/go-json v0.0.0-20230131223807-18775e0fb4fb // indirect
@@ -105,9 +154,9 @@ require (
 	github.com/fatih/color v1.17.0 // indirect
 	github.com/fxamacker/cbor/v2 v2.7.0 // indirect
 	github.com/ghodss/yaml v1.0.1-0.20190212211648-25d852aebe32 // indirect
-	github.com/go-openapi/jsonpointer v0.20.0 // indirect
-	github.com/go-openapi/jsonreference v0.20.2 // indirect
-	github.com/go-openapi/swag v0.22.4 // indirect
+	github.com/go-openapi/jsonpointer v0.20.2 // indirect
+	github.com/go-openapi/jsonreference v0.20.4 // indirect
+	github.com/go-openapi/swag v0.22.9 // indirect
 	github.com/gobuffalo/flect v1.0.2 // indirect
 	github.com/google/gnostic-models v0.6.8 // indirect
 	github.com/gookit/color v1.5.4 // indirect
@@ -123,6 +172,7 @@ require (
 	github.com/onsi/ginkgo/v2 v2.20.0 // indirect
 	github.com/openshift/client-go v0.0.0-20240528061634-b054aa794d87 // indirect
 	github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect
+	github.com/prometheus/prometheus v0.54.1
 	github.com/rivo/uniseg v0.4.7 // indirect
 	github.com/robfig/cron v1.2.0 // indirect
 	github.com/ssor/bom v0.0.0-20170718123548-6386211fdfcf // indirect