From 7b40fb76f696f74d588a311a31f5b87e8f179400 Mon Sep 17 00:00:00 2001
From: Fabian Kramm <fab.kramm@googlemail.com>
Date: Mon, 5 Aug 2024 10:46:26 +0200
Subject: [PATCH] fix: metrics server hpa problem

---
 pkg/apiservice/generic.go                     | 64 +++++++++++++------
 .../metricsserver/metricsserver.go            | 11 +++-
 2 files changed, 55 insertions(+), 20 deletions(-)

diff --git a/pkg/apiservice/generic.go b/pkg/apiservice/generic.go
index ca5aa0ebc..e16001463 100644
--- a/pkg/apiservice/generic.go
+++ b/pkg/apiservice/generic.go
@@ -18,8 +18,11 @@ import (
 	"k8s.io/apimachinery/pkg/runtime/schema"
 	"k8s.io/apimachinery/pkg/runtime/serializer"
 	"k8s.io/apimachinery/pkg/types"
+	"k8s.io/apimachinery/pkg/util/sets"
 	"k8s.io/apimachinery/pkg/util/wait"
+	genericapifilters "k8s.io/apiserver/pkg/endpoints/filters"
 	"k8s.io/apiserver/pkg/endpoints/handlers/responsewriters"
+	"k8s.io/apiserver/pkg/endpoints/request"
 	"k8s.io/client-go/rest"
 	"k8s.io/klog/v2"
 	apiregistrationv1 "k8s.io/kube-aggregator/pkg/apis/apiregistration/v1"
@@ -139,7 +142,14 @@ func createOperation(ctrlCtx *synccontext.ControllerContext, serviceName string,
 	}
 }
 
-func StartAPIServiceProxy(ctx *synccontext.ControllerContext, targetServiceName, targetServiceNamespace string, targetPort, hostPort int) error {
+func StartAPIServiceProxy(
+	ctx *synccontext.ControllerContext,
+	targetServiceName,
+	targetServiceNamespace string,
+	targetPort,
+	hostPort int,
+	extraHandlers ...func(h http.Handler) http.Handler,
+) error {
 	tlsCertFile := ctx.Config.VirtualClusterKubeConfig().ServerCACert
 	tlsKeyFile := ctx.Config.VirtualClusterKubeConfig().ServerCAKey
 
@@ -162,25 +172,21 @@ func StartAPIServiceProxy(ctx *synccontext.ControllerContext, targetServiceName,
 		return fmt.Errorf("create host proxy handler: %w", err)
 	}
 
-	s := serializer.NewCodecFactory(scheme.Scheme)
-	server := &http.Server{
-		Addr: "localhost:" + strconv.Itoa(hostPort),
-		Handler: http.HandlerFunc(func(writer http.ResponseWriter, request *http.Request) {
-			// we only allow traffic to discovery paths
-			if !isAPIServiceProxyPathAllowed(request.Method, request.URL.Path) {
-				klog.FromContext(ctx).Info("Denied access to api service proxy at path", "path", request.URL.Path, "method", request.Method)
-				responsewriters.ErrorNegotiated(
-					kerrors.NewForbidden(metav1.SchemeGroupVersion.WithResource("proxy").GroupResource(), "proxy", fmt.Errorf("paths other than discovery paths are not allowed")),
-					s,
-					corev1.SchemeGroupVersion,
-					writer,
-					request,
-				)
-				return
-			}
+	h := serveHandler(ctx, proxyHandler)
+
+	// add custom handlers
+	for _, extraHandler := range extraHandlers {
+		h = extraHandler(h)
+	}
+
+	h = genericapifilters.WithRequestInfo(h, &request.RequestInfoFactory{
+		APIPrefixes:          sets.NewString("api", "apis"),
+		GrouplessAPIPrefixes: sets.NewString("api"),
+	})
 
-			proxyHandler.ServeHTTP(writer, request)
-		}),
+	server := &http.Server{
+		Addr:    "localhost:" + strconv.Itoa(hostPort),
+		Handler: h,
 	}
 
 	go func() {
@@ -195,6 +201,26 @@ func StartAPIServiceProxy(ctx *synccontext.ControllerContext, targetServiceName,
 	return nil
 }
 
+func serveHandler(ctx context.Context, next http.Handler) http.Handler {
+	s := serializer.NewCodecFactory(scheme.Scheme)
+	return http.HandlerFunc(func(writer http.ResponseWriter, request *http.Request) {
+		// we only allow traffic to discovery paths
+		if !isAPIServiceProxyPathAllowed(request.Method, request.URL.Path) {
+			klog.FromContext(ctx).Info("Denied access to api service proxy at path", "path", request.URL.Path, "method", request.Method)
+			responsewriters.ErrorNegotiated(
+				kerrors.NewForbidden(metav1.SchemeGroupVersion.WithResource("proxy").GroupResource(), "proxy", fmt.Errorf("paths other than discovery paths are not allowed")),
+				s,
+				corev1.SchemeGroupVersion,
+				writer,
+				request,
+			)
+			return
+		}
+
+		next.ServeHTTP(writer, request)
+	})
+}
+
 func isAPIServiceProxyPathAllowed(method, path string) bool {
 	if strings.ToUpper(method) != http.MethodGet {
 		return false
diff --git a/pkg/integrations/metricsserver/metricsserver.go b/pkg/integrations/metricsserver/metricsserver.go
index 366879c8c..86311c8fa 100644
--- a/pkg/integrations/metricsserver/metricsserver.go
+++ b/pkg/integrations/metricsserver/metricsserver.go
@@ -52,7 +52,16 @@ func Register(ctx *synccontext.ControllerContext) error {
 		targetService := cmp.Or(ctx.Config.Integrations.MetricsServer.APIService.Service.Name, "metrics-server")
 		targetServiceNamespace := cmp.Or(ctx.Config.Integrations.MetricsServer.APIService.Service.Namespace, "kube-system")
 		targetServicePort := cmp.Or(ctx.Config.Integrations.MetricsServer.APIService.Service.Port, 443)
-		err := apiservice.StartAPIServiceProxy(ctx, targetService, targetServiceNamespace, targetServicePort, hostPort)
+		err := apiservice.StartAPIServiceProxy(
+			ctx,
+			targetService,
+			targetServiceNamespace,
+			targetServicePort,
+			hostPort,
+			func(h http.Handler) http.Handler {
+				return WithMetricsServerProxy(h, ctx.ToRegisterContext())
+			},
+		)
 		if err != nil {
 			return fmt.Errorf("start api service proxy: %w", err)
 		}