feat: add metrics for konnect entity operations

CHANGELOG.md

@@ -37,6 +37,13 @@
   and preview deployments are now customized with Konnect-related settings.
   [#910](https://github.com/Kong/gateway-operator/pull/910)
 
+### Added
+
+- Added prometheus metrics for Konnect entity operations in the metrics server:
+  - `gateway_operator_konnect_entity_operation_count` for number of operations.
+  - `gateway_operator_konnect_entity_operation_duration_milliseconds` for duration of operations.
+  [#953](https://github.com/Kong/gateway-operator/pull/953)
+
 ## [v1.4.1]
 
 > Release date: 2024-11-28

controller/konnect/ops/ops.go

@@ -16,6 +16,7 @@ import (
 	"github.com/kong/gateway-operator/controller/konnect/constraints"
 	sdkops "github.com/kong/gateway-operator/controller/konnect/ops/sdk"
 	"github.com/kong/gateway-operator/controller/pkg/log"
+	"github.com/kong/gateway-operator/internal/metrics"
 	"github.com/kong/gateway-operator/pkg/consts"
 	k8sutils "github.com/kong/gateway-operator/pkg/utils/kubernetes"
 
@@ -45,11 +46,15 @@ func Create[
 	ctx context.Context,
 	sdk sdkops.SDKWrapper,
 	cl client.Client,
+	metricRecorder metrics.Recorder,
 	e TEnt,
 ) (*T, error) {
 	var (
 		err   error
 		start = time.Now()
+
+		entityType = e.GetTypeName()
+		statusCode int
 	)
 	switch ent := any(e).(type) {
 	case *konnectv1alpha1.KonnectGatewayControlPlane:
@@ -162,6 +167,7 @@ func Create[
 		}
 
 	case errors.As(err, &errSDK):
+		statusCode = errSDK.StatusCode
 		SetKonnectEntityProgrammedConditionFalse(e, consts.KonnectEntitiesFailedToCreateReason, errSDK.Error())
 	case errors.As(err, &errRelationsFailed):
 		e.SetKonnectID(errRelationsFailed.KonnectID)
@@ -172,6 +178,22 @@ func Create[
 		SetKonnectEntityProgrammedCondition(e)
 	}
 
+	if err != nil {
+		metricRecorder.RecordKonnectEntityOperationFailure(
+			sdk.GetServerURL(),
+			metrics.KonnectEntityOperationCreate,
+			entityType,
+			time.Since(start),
+			statusCode,
+		)
+	} else {
+		metricRecorder.RecordKonnectEntityOperationSuccess(
+			sdk.GetServerURL(),
+			metrics.KonnectEntityOperationCreate,
+			entityType,
+			time.Since(start),
+		)
+	}
 	logOpComplete(ctx, start, CreateOp, e, err)
 
 	return e, IgnoreUnrecoverableAPIErr(err, loggerForEntity(ctx, e, CreateOp))
@@ -182,7 +204,7 @@ func Create[
 func Delete[
 	T constraints.SupportedKonnectEntityType,
 	TEnt constraints.EntityType[T],
-](ctx context.Context, sdk sdkops.SDKWrapper, cl client.Client, ent TEnt) error {
+](ctx context.Context, sdk sdkops.SDKWrapper, cl client.Client, metricRecorder metrics.Recorder, ent TEnt) error {
 	if ent.GetKonnectStatus().GetKonnectID() == "" {
 		cond, ok := k8sutils.GetCondition(konnectv1alpha1.KonnectEntityProgrammedConditionType, ent)
 		if ok && cond.Status == metav1.ConditionTrue {
@@ -197,6 +219,9 @@ func Delete[
 	var (
 		err   error
 		start = time.Now()
+
+		entityType = ent.GetTypeName()
+		statusCode int
 	)
 	switch ent := any(ent).(type) {
 	case *konnectv1alpha1.KonnectGatewayControlPlane:
@@ -245,6 +270,26 @@ func Delete[
 		return fmt.Errorf("unsupported entity type %T", ent)
 	}
 
+	if err != nil {
+		var errSDK *sdkkonnecterrs.SDKError
+		if errors.As(err, &errSDK) {
+			statusCode = errSDK.StatusCode
+		}
+		metricRecorder.RecordKonnectEntityOperationFailure(
+			sdk.GetServerURL(),
+			metrics.KonnectEntityOperationDelete,
+			entityType,
+			time.Since(start),
+			statusCode,
+		)
+	} else {
+		metricRecorder.RecordKonnectEntityOperationSuccess(
+			sdk.GetServerURL(),
+			metrics.KonnectEntityOperationDelete,
+			entityType,
+			time.Since(start),
+		)
+	}
 	logOpComplete(ctx, start, DeleteOp, ent, err)
 
 	return err
@@ -297,6 +342,7 @@ func Update[
 	sdk sdkops.SDKWrapper,
 	syncPeriod time.Duration,
 	cl client.Client,
+	metricRecorder metrics.Recorder,
 	e TEnt,
 ) (ctrl.Result, error) {
 	now := time.Now()
@@ -312,7 +358,13 @@ func Update[
 		)
 	}
 
-	var err error
+	var (
+		err error
+
+		entityType = e.GetTypeName()
+		statusCode int
+		start      = time.Now()
+	)
 	switch ent := any(e).(type) {
 	case *konnectv1alpha1.KonnectGatewayControlPlane:
 		err = updateControlPlane(ctx, sdk.GetControlPlaneSDK(), sdk.GetControlPlaneGroupSDK(), cl, ent)
@@ -367,6 +419,7 @@ func Update[
 	)
 	switch {
 	case errors.As(err, &errSDK):
+		statusCode = errSDK.StatusCode
 		SetKonnectEntityProgrammedConditionFalse(e, consts.KonnectEntitiesFailedToUpdateReason, errSDK.Body)
 	case errors.As(err, &errRelationsFailed):
 		e.SetKonnectID(errRelationsFailed.KonnectID)
@@ -377,7 +430,23 @@ func Update[
 		SetKonnectEntityProgrammedCondition(e)
 	}
 
-	logOpComplete(ctx, now, UpdateOp, e, err)
+	if err != nil {
+		metricRecorder.RecordKonnectEntityOperationFailure(
+			sdk.GetServerURL(),
+			metrics.KonnectEntityOperationUpdate,
+			entityType,
+			time.Since(start),
+			statusCode,
+		)
+	} else {
+		metricRecorder.RecordKonnectEntityOperationSuccess(
+			sdk.GetServerURL(),
+			metrics.KonnectEntityOperationUpdate,
+			entityType,
+			time.Since(start),
+		)
+	}
+	logOpComplete(ctx, start, UpdateOp, e, err)
 
 	return ctrl.Result{}, IgnoreUnrecoverableAPIErr(err, loggerForEntity(ctx, e, UpdateOp))
 }

controller/konnect/ops/ops_controlplane_test.go

@@ -21,6 +21,7 @@ import (
 	"sigs.k8s.io/controller-runtime/pkg/client/fake"
 
 	sdkmocks "github.com/kong/gateway-operator/controller/konnect/ops/sdk/mocks"
+	"github.com/kong/gateway-operator/internal/metrics"
 	"github.com/kong/gateway-operator/modules/manager/scheme"
 	"github.com/kong/gateway-operator/pkg/consts"
 
@@ -619,7 +620,7 @@ func TestCreateAndUpdateControlPlane_KubernetesMetadataConsistency(t *testing.T)
 				ID: "12345",
 			},
 		}, nil)
-	_, err := Create(ctx, sdk.SDK, fakeClient, cp)
+	_, err := Create(ctx, sdk.SDK, fakeClient, &metrics.MockRecorder{}, cp)
 	require.NoError(t, err)
 
 	t.Log("Triggering UpdateControlPlane with expected labels")
@@ -633,7 +634,7 @@ func TestCreateAndUpdateControlPlane_KubernetesMetadataConsistency(t *testing.T)
 				ID: "12345",
 			},
 		}, nil)
-	_, err = Update(ctx, sdk.SDK, 0, fakeClient, cp)
+	_, err = Update(ctx, sdk.SDK, 0, fakeClient, &metrics.MockRecorder{}, cp)
 	require.NoError(t, err)
 }
 

controller/konnect/ops/ops_test.go

@@ -17,6 +17,7 @@ import (
 
 	"github.com/kong/gateway-operator/controller/konnect/constraints"
 	sdkmocks "github.com/kong/gateway-operator/controller/konnect/ops/sdk/mocks"
+	"github.com/kong/gateway-operator/internal/metrics"
 	"github.com/kong/gateway-operator/modules/manager/scheme"
 	"github.com/kong/gateway-operator/pkg/consts"
 
@@ -237,7 +238,7 @@ func testCreate[
 				sdk = tc.sdkFunc(t, sdk)
 			}
 
-			_, err := Create(context.Background(), sdk, fakeClient, tc.entity)
+			_, err := Create(context.Background(), sdk, fakeClient, &metrics.MockRecorder{}, tc.entity)
 			if tc.expectedErrorContains != "" {
 				require.ErrorContains(t, err, tc.expectedErrorContains)
 			} else {
@@ -345,7 +346,7 @@ func testDelete[
 				sdk = tc.sdkFunc(t, sdk)
 			}
 
-			err := Delete(context.Background(), sdk, fakeClient, tc.entity)
+			err := Delete(context.Background(), sdk, fakeClient, &metrics.MockRecorder{}, tc.entity)
 			if tc.expectedError != "" {
 				require.ErrorContains(t, err, tc.expectedError)
 				return

controller/konnect/ops/sdk/mocks/sdkfactory_mock.go

@@ -62,6 +62,14 @@ func NewMockSDKWrapperWithT(t *testing.T) *MockSDKWrapper {
 	}
 }
 
+const (
+	mockSDKServerURL = "http://mock-api.konnect.test"
+)
+
+func (m MockSDKWrapper) GetServerURL() string {
+	return mockSDKServerURL
+}
+
 func (m MockSDKWrapper) GetControlPlaneSDK() sdkops.ControlPlaneSDK {
 	return m.ControlPlaneSDK
 }

controller/konnect/ops/sdk/sdkfactory.go

@@ -29,14 +29,23 @@ type SDKWrapper interface {
 	GetKeySetsSDK() KeySetsSDK
 	GetSNIsSDK() SNIsSDK
 	GetDataPlaneCertificatesSDK() DataPlaneClientCertificatesSDK
+
+	// GetServerURL returns the server URL for recording metrics.
+	GetServerURL() string
 }
 
 type sdkWrapper struct {
-	sdk *sdkkonnectgo.SDK
+	serverURL string
+	sdk       *sdkkonnectgo.SDK
 }
 
 var _ SDKWrapper = sdkWrapper{}
 
+// GetServerURL returns the Konnect server URL for recording metrics.
+func (w sdkWrapper) GetServerURL() string {
+	return w.serverURL
+}
+
 // GetControlPlaneSDK returns the SDK to operate Konnect control planes.
 func (w sdkWrapper) GetControlPlaneSDK() ControlPlaneSDK {
 	return w.sdk.ControlPlanes
@@ -165,6 +174,7 @@ func NewSDKFactory() SDKFactory {
 // NewKonnectSDK creates a new Konnect SDK.
 func (f sdkFactory) NewKonnectSDK(serverURL string, token SDKToken) SDKWrapper {
 	return sdkWrapper{
+		serverURL: serverURL,
 		sdk: sdkkonnectgo.New(
 			sdkkonnectgo.WithSecurity(
 				sdkkonnectcomp.Security{

controller/konnect/reconciler_generic.go

@@ -22,6 +22,7 @@ import (
 	sdkops "github.com/kong/gateway-operator/controller/konnect/ops/sdk"
 	"github.com/kong/gateway-operator/controller/pkg/log"
 	"github.com/kong/gateway-operator/controller/pkg/patch"
+	"github.com/kong/gateway-operator/internal/metrics"
 	"github.com/kong/gateway-operator/pkg/consts"
 	k8sutils "github.com/kong/gateway-operator/pkg/utils/kubernetes"
 
@@ -45,6 +46,8 @@ type KonnectEntityReconciler[T constraints.SupportedKonnectEntityType, TEnt cons
 	Client                  client.Client
 	SyncPeriod              time.Duration
 	MaxConcurrentReconciles uint
+
+	MetricRecoder metrics.Recorder
 }
 
 // KonnectEntityReconcilerOption is a functional option for the KonnectEntityReconciler.
@@ -71,6 +74,15 @@ func WithKonnectMaxConcurrentReconciles[T constraints.SupportedKonnectEntityType
 	}
 }
 
+// WithMetricRecoder sets the metric recorder to record metrics of Konnect entity operations of the reconciler.
+func WithMetricRecorder[T constraints.SupportedKonnectEntityType, TEnt constraints.EntityType[T]](
+	metricRecorder metrics.Recorder,
+) KonnectEntityReconcilerOption[T, TEnt] {
+	return func(r *KonnectEntityReconciler[T, TEnt]) {
+		r.MetricRecoder = metricRecorder
+	}
+}
+
 // NewKonnectEntityReconciler returns a new KonnectEntityReconciler for the given
 // Konnect entity type.
 func NewKonnectEntityReconciler[
@@ -88,6 +100,7 @@ func NewKonnectEntityReconciler[
 		Client:                  client,
 		SyncPeriod:              consts.DefaultKonnectSyncPeriod,
 		MaxConcurrentReconciles: consts.DefaultKonnectMaxConcurrentReconciles,
+		MetricRecoder:           &metrics.MockRecorder{},
 	}
 	for _, opt := range opts {
 		opt(r)
@@ -469,7 +482,7 @@ func (r *KonnectEntityReconciler[T, TEnt]) Reconcile(
 		}
 
 		if controllerutil.RemoveFinalizer(ent, KonnectCleanupFinalizer) {
-			if err := ops.Delete[T, TEnt](ctx, sdk, r.Client, ent); err != nil {
+			if err := ops.Delete[T, TEnt](ctx, sdk, r.Client, r.MetricRecoder, ent); err != nil {
 				if res, errStatus := patch.StatusWithCondition(
 					ctx, r.Client, ent,
 					konnectv1alpha1.KonnectEntityProgrammedConditionType,
@@ -500,7 +513,7 @@ func (r *KonnectEntityReconciler[T, TEnt]) Reconcile(
 	// https://github.com/kubernetes/kubernetes/blob/master/pkg/controller/controller_utils.go
 	if status := ent.GetKonnectStatus(); status == nil || status.GetKonnectID() == "" {
 		obj := ent.DeepCopyObject().(client.Object)
-		_, err := ops.Create[T, TEnt](ctx, sdk, r.Client, ent)
+		_, err := ops.Create[T, TEnt](ctx, sdk, r.Client, r.MetricRecoder, ent)
 
 		// TODO: this is actually not 100% error prone because when status
 		// update fails we don't store the Konnect ID and hence the reconciler
@@ -557,7 +570,7 @@ func (r *KonnectEntityReconciler[T, TEnt]) Reconcile(
 		return ctrl.Result{}, nil
 	}
 
-	if res, err := ops.Update[T, TEnt](ctx, sdk, r.SyncPeriod, r.Client, ent); err != nil {
+	if res, err := ops.Update[T, TEnt](ctx, sdk, r.SyncPeriod, r.Client, r.MetricRecoder, ent); err != nil {
 		setServerURLAndOrgID(ent, serverURL, apiAuth.Status.OrganizationID)
 		if errUpd := r.Client.Status().Update(ctx, ent); errUpd != nil {
 			if k8serrors.IsConflict(errUpd) {