Introduce autoscalers for inbox listener and stream service

charts/primary-site/templates/cronjobs/garbage-collector.yaml → charts/primary-site/templates/garbage-collector-cronjob.yaml

@@ -1,9 +1,9 @@
 apiVersion: batch/v1
 kind: CronJob
 metadata:
-  name: garbage-collector
+  name: {{ .Values.garbageCollector.name }}
   labels:
-    app: garbage-collector
+    app: {{ .Values.garbageCollector.name }}
 spec:
   concurrencyPolicy: Forbid
   schedule: "{{ .Values.garbageCollector.schedule }}"
@@ -19,7 +19,7 @@ spec:
                 secretName: gcp-cloud-credential
                 optional: true
           containers:
-            - name: garbage-collector
+            - name: {{ .Values.garbageCollector.name }}
               image: us-central1-docker.pkg.dev/foxglove-images/images/garbage-collector:{{ .Chart.AppVersion }}
               volumeMounts:
                 - mountPath: /secrets
@@ -29,8 +29,6 @@ spec:
                     name: cloud-credentials
                     optional: true
               env:
-                - name: GOOGLE_APPLICATION_CREDENTIALS
-                  value: /secrets/credentials.json
                 - name: FOXGLOVE_API_URL
                   value: "{{ .Values.globals.foxgloveApiUrl }}"
                 - name: FOXGLOVE_SITE_TOKEN
@@ -47,4 +45,8 @@ spec:
                   value: "{{ .Values.globals.azure.storageAccountName }}"
                 - name: STORAGE_AZURE_SERVICE_URL
                   value: "{{ .Values.globals.azure.serviceUrl }}"
+                {{- range $item := .Values.inboxListener.env }}
+                - name: {{ $item.value }}
+                  value: {{ $item.value }}
+                {{- end }}
           restartPolicy: OnFailure

charts/primary-site/templates/deployments/inbox-listener.yaml → charts/primary-site/templates/inbox-listener-deployment.yaml

@@ -1,13 +1,13 @@
 apiVersion: apps/v1
 kind: Deployment
 metadata:
-  name: inbox-listener
+  name: {{ .Values.inboxListener.name }}
   labels:
-    app: inbox-listener
+    app: {{ .Values.inboxListener.name }}
 spec:
   selector:
     matchLabels:
-      app: inbox-listener
+      app: {{ .Values.inboxListener.name }}
   strategy:
     rollingUpdate:
       maxSurge: 25%
@@ -16,15 +16,15 @@ spec:
   template:
     metadata:
       labels:
-        app: inbox-listener
+        app: {{ .Values.inboxListener.name }}
     spec:
       volumes:
         - name: cloud-credentials
           secret:
             secretName: gcp-cloud-credential
             optional: true
       containers:
-        - name: inbox-listener
+        - name: {{ .Values.inboxListener.name }}
           image: us-central1-docker.pkg.dev/foxglove-images/images/inbox-listener:{{ .Chart.AppVersion }}
           resources:
             requests:
@@ -41,8 +41,6 @@ spec:
                 name: cloud-credentials
                 optional: true
           env:
-            - name: GOOGLE_APPLICATION_CREDENTIALS
-              value: /secrets/credentials.json
             - name: FOXGLOVE_API_URL
               value: "{{ .Values.globals.foxgloveApiUrl }}"
             - name: FOXGLOVE_SITE_TOKEN
@@ -65,3 +63,7 @@ spec:
               value: "{{ .Values.globals.azure.storageAccountName }}"
             - name: STORAGE_AZURE_SERVICE_URL
               value: "{{ .Values.globals.azure.serviceUrl }}"
+            {{- range $item := .Values.inboxListener.env }}
+            - name: {{ $item.value }}
+              value: {{ $item.value }}
+            {{- end }}

charts/primary-site/templates/inbox-listener-scaledobject.yaml

@@ -0,0 +1,34 @@
+apiVersion: keda.sh/v1alpha1
+kind: TriggerAuthentication
+metadata:
+  name: keda-inbox-listener-auth
+spec:
+  secretTargetRef:
+    - parameter: token
+      name: foxglove-site
+      key: token
+---
+apiVersion: keda.sh/v1alpha1
+kind: ScaledObject
+metadata:
+  name: inbox-listener-scaledobject
+  labels:
+spec:
+  scaleTargetRef:
+    apiVersion: apps/v1
+    kind: deployment
+    name: {{ .Values.inboxListener.name }}
+    envSourceContainerName: {{ .Values.inboxListener.name }}
+  pollingInterval: {{ .Values.inboxListener.deployment.autoscalePollingInterval }}
+  cooldownPeriod: {{ .Values.inboxListener.deployment.autoscaleCooldownPeriod }}
+  minReplicaCount: {{ .Values.inboxListener.deployment.minReplicas }}
+  maxReplicaCount: {{ .Values.inboxListener.deployment.maxReplicas }}
+  triggers:
+    - type: metrics-api
+      metadata:
+        targetValue: "{{ .Values.inboxListener.deployment.scaling.targetQueueDepth }}"
+        url: "{{ .Values.globals.foxgloveApiUrl }}/internal/platform/v1/pending-imports-stats"
+        valueLocation: 'unleased'
+        authMode: "bearer"
+      authenticationRef:
+        name: keda-inbox-listener-auth

charts/primary-site/templates/deployments/stream-service.yaml → charts/primary-site/templates/stream-service-deployment.yaml

@@ -1,14 +1,14 @@
 apiVersion: apps/v1
 kind: Deployment
 metadata:
-  name: stream-service
+  name: {{ .Values.streamService.name }}
   labels:
-    app: stream-service
+    app: {{ .Values.streamService.name }}
 spec:
   replicas: 1
   selector:
     matchLabels:
-      app: stream-service
+      app: {{ .Values.streamService.name }}
   strategy:
     rollingUpdate:
       maxSurge: 25%
@@ -17,15 +17,15 @@ spec:
   template:
     metadata:
       labels:
-        app: stream-service
+        app: {{ .Values.streamService.name }}
     spec:
       volumes:
         - name: cloud-credentials
           secret:
             secretName: gcp-cloud-credential
             optional: true
       containers:
-        - name: stream-service
+        - name: {{ .Values.streamService.name }}
           image: us-central1-docker.pkg.dev/foxglove-images/images/stream-service:{{ .Chart.AppVersion }}
           resources:
             requests:
@@ -44,8 +44,6 @@ spec:
                 name: cloud-credentials
                 optional: true
           env:
-            - name: GOOGLE_APPLICATION_CREDENTIALS
-              value: /secrets/credentials.json
             - name: FOXGLOVE_API_URL
               value: "{{ .Values.globals.foxgloveApiUrl }}"
             - name: PORT
@@ -56,6 +54,10 @@ spec:
               value: "{{ .Values.globals.azure.storageAccountName }}"
             - name: STORAGE_AZURE_SERVICE_URL
               value: "{{ .Values.globals.azure.serviceUrl }}"
+            {{- range $item := .Values.inboxListener.env }}
+            - name: {{ $item.value }}
+              value: {{ $item.value }}
+            {{- end }}
           readinessProbe:
             httpGet:
               path: /liveness

charts/primary-site/templates/stream-service-hpa.yaml

@@ -0,0 +1,19 @@
+apiVersion: autoscaling/v2beta2
+kind: HorizontalPodAutoscaler
+metadata:
+  name: {{ .Values.streamService.name }}
+spec:
+  scaleTargetRef:
+    apiVersion: apps/v1
+    kind: Deployment
+    name: {{ .Values.streamService.name }}
+  minReplicas: {{ .Values.streamService.deployment.scaling.minReplicas }}
+  maxReplicas: {{ .Values.streamService.deployment.scaling.maxReplicas }}
+  metrics:
+    - type: Resource
+      resource:
+        name: cpu
+        target:
+          type: Utilization
+          averageUtilization: {{ .Values.streamService.deployment.scaling.targetCPUUtilization }}
+

charts/primary-site/values.yaml

@@ -1,7 +1,6 @@
 globals:
   siteToken:
   foxgloveApiUrl: https://api.foxglove.dev
-
   ## Supported storageProvider values are: `google_cloud` or `azure`
   ## If `azure` is used, then the `@azure.storageAccountName` and `@azure.serviceUrl` values
   ## are required.
@@ -11,33 +10,48 @@ globals:
   inbox:
     storageProvider: google_cloud
     bucketName: foxglove-inbox
-
   azure:
     storageAccountName: ""
     ## For example: https://<resourcegroup>.blob.core.windows.net
     serviceUrl: ""
 
 inboxListener:
+  name: inbox-listener
   deployment:
+    scaling:
+      autoscalePollingInterval: 10
+      autoscaleCooldownPeriod: 10
+      minReplicas: 1
+      maxReplicas: 10
+      targetQueueDepth: 2
     resources:
       requests:
         cpu: 1000m
         memory: 1Gi
       limits:
         cpu: 1000m
         memory: 1Gi
+  env:
 
 streamService:
+  name: stream-service
   deployment:
+    scaling:
+      minReplicas: 1
+      maxReplicas: 3
+      targetCPUUtilization: 80
     resources:
       requests:
         cpu: 1000m
         memory: 1Gi
       limits:
         cpu: 1000m
         memory: 1Gi
+  env:
 
 garbageCollector:
+  name: garbage-collector
   schedule: "*/10 * * * *" # every 10 minutes
   failedJobsHistoryLimit: 1
   successfulJobsHistoryLimit: 3
+  env: