feat(server): support resource monitor (#1468)

* feat(server): support resource monitor

* test(server): add monitor test

* fix(server): get app namespace by GetApplicationNamespaceByAppId

* chore: change prometheus conf

* chore(build): add prometheus

* fix: build

* chore

.vscode/settings.json

@@ -134,5 +134,6 @@
   "i18n-ally.keystyle": "nested",
   "i18n-ally.keysInUse": [
     "description.part2_whatever"
-  ]
+  ],
+  "jest.rootPath": "e2e",
 }

build/charts/laf-server/templates/deployment.yaml

@@ -89,6 +89,8 @@ spec:
               value: {{ .Values.default_region.log_server_secret }}
             - name: DEFAULT_REGION_LOG_SERVER_DATABASE_URL
               value: {{ .Values.default_region.log_server_database_url }}
+            - name: DEFAULT_REGION_PROMETHEUS_URL
+              value: {{ .Values.default_region.prometheus_url }}
             - name: SITE_NAME
               value: {{ .Values.siteName | quote}}
       {{- with .Values.nodeSelector }}

build/charts/minio/values.yaml

@@ -510,7 +510,7 @@ metrics:
   serviceMonitor:
     enabled: false
     # scrape each node/pod individually for additional metrics
-    includeNode: false 
+    includeNode: false
     public: true
     additionalLabels: {}
     # for node metrics
@@ -521,8 +521,8 @@ metrics:
       #   - regex: (server|pod)
       #     action: labeldrop
     # namespace: monitoring
-    # interval: 30s
-    # scrapeTimeout: 10s
+    interval: 30s
+    scrapeTimeout: 10s
 
 ## ETCD settings: https://github.com/minio/minio/blob/master/docs/sts/etcd.md
 ## Define endpoints to enable this section.

build/prometheus-helm.yaml

@@ -0,0 +1,25 @@
+fullnameOverride: prometheus
+alertmanager:
+  enabled: false
+grafana:
+  enabled: false
+coreDns:
+  enabled: false
+nodeExporter:
+  enabled: false
+kubeApiServer:
+  enabled: false
+kubeScheduler:
+  enabled: false
+kubeControllerManager:
+  enabled: false
+kubeEtcd:
+  enabled: false
+kubeProxy:
+  enabled: false
+kubeStateMetrics:
+  enabled: false
+prometheus:
+  networkPolicy:
+    enabled: true
+    ingress: []

build/start.sh

@@ -18,6 +18,8 @@ ENABLE_APISIX_HOST_NETWORK=${ENABLE_APISIX_HOST_NETWORK:-true}
 NAMESPACE=${NAMESPACE:-laf-system}
 PASSWD_OR_SECRET=$(tr -cd 'a-z0-9' </dev/urandom |head -c32)
 
+ENABLE_MONITOR=${ENABLE_MONITOR:-true}
+
 # *************** Deployments **************** #
 
 ## 0. create namespace
@@ -70,10 +72,32 @@ helm install minio -n ${NAMESPACE} \
     --set persistence.size=${OSS_PV_SIZE:-3Gi} \
     --set domain=${MINIO_DOMAIN} \
     --set consoleHost=minio.${DOMAIN} \
+    --set metrics.serviceMonitor.enabled=${ENABLE_MONITOR} \
+    --set metrics.serviceMonitor.additionalLabels.release=prometheus \
+    --set metrics.serviceMonitor.additionalLabels.namespace=${NAMESPACE} \
     ./charts/minio
 
+## 4. install prometheus
+PROMETHEUS_URL=http://prometheus-prometheus.${NAMESPACE}.svc.cluster.local:9090
+if [ $ENABLE_MONITOR ]; then
+    helm repo add prometheus-community https://prometheus-community.github.io/helm-charts
+    helm repo update
+
+    sed "s/\$NAMESPACE/$NAMESPACE/g" prometheus-helm.yaml > prometheus-helm-with-values.yaml
+
+    helm install prometheus -n ${NAMESPACE} \
+        -f ./prometheus-helm-with-values.yaml \
+        prometheus-community/kube-prometheus-stack
+
+    helm install prometheus-mongodb-exporter -n ${NAMESPACE} \
+        --set mongodb.uri=${DATABASE_URL} \
+        --set serviceMonitor.enabled=true \
+        --set serviceMonitor.additionalLabels.release=prometheus \
+        --set serviceMonitor.additionalLabels.namespace=${NAMESPACE} \
+        prometheus-community/prometheus-mongodb-exporter
+fi
 
-## 4. install laf-server
+## 5. install laf-server
 SERVER_JWT_SECRET=$PASSWD_OR_SECRET
 LOG_SERVER_URL="http://log-server.${NAMESPACE}.svc.cluster.local:5060"
 LOG_SERVER_DATABASE_URL="mongodb://${DB_USERNAME:-admin}:${PASSWD_OR_SECRET}@mongodb-0.mongo.${NAMESPACE}.svc.cluster.local:27017/function-logs?authSource=admin&replicaSet=rs0&w=majority"
@@ -100,6 +124,7 @@ helm install server -n ${NAMESPACE} \
     --set default_region.log_server_url=${LOG_SERVER_URL} \
     --set default_region.log_server_secret=${LOG_SERVER_SECRET} \
     --set default_region.log_server_database_url=${LOG_SERVER_DATABASE_URL} \
+    $( [[ $ENABLE_MONITOR ]] && echo "--set default_region.prometheus_url=${PROMETHEUS_URL}" ) \
     ./charts/laf-server
 
 ## 6. install laf-web

e2e/.gitignore

@@ -0,0 +1 @@
+.env

e2e/.prettierrc

@@ -0,0 +1,5 @@
+{
+  "singleQuote": true,
+  "trailingComma": "all",
+  "semi": false
+}

e2e/2-monitor/00-query.test.ts

@@ -0,0 +1,104 @@
+import { describe, expect, test } from '@jest/globals'
+import { api, EnsureTestToken, GetRegion, GetRuntime } from '../api'
+import { Config } from '../config'
+
+describe('query monitor metrics normally', () => {
+  let appid = Config.TEST_APPID
+  let token = null
+  beforeAll(async () => {
+    token = await EnsureTestToken()
+    expect(token).toBeTruthy()
+  })
+
+  test(
+    'query monitor metrics',
+    async () => {
+      const metrics = [
+        'cpuUsage',
+        'memoryUsage',
+        'storageUsage',
+        'databaseUsage',
+      ]
+
+      const query = {
+        q: metrics,
+        step: 300,
+      }
+
+      const res = await api.get(`/v1/monitor/${appid}/metrics`, {
+        params: query,
+        headers: { Authorization: `Bearer ${token}` },
+      })
+
+      expect(res.status).toBe(200)
+
+      const data = res.data?.data
+      expect(Object.keys(data)).toEqual(expect.arrayContaining(metrics))
+
+      Object.values(data).forEach((v: []) => {
+        expect(Array.isArray(v)).toBeTruthy()
+        v.forEach((item) => {
+          expect([
+            ['values', 'metric'],
+            ['value', 'metric'],
+          ]).toContainEqual(expect.arrayContaining(Object.keys(item)))
+        })
+      })
+    },
+    10 * 1000,
+  )
+})
+
+describe('query monitor metrics with invalid inputs', () => {
+  let appid = Config.TEST_APPID
+  let token = null
+  beforeAll(async () => {
+    token = await EnsureTestToken()
+    expect(token).toBeTruthy()
+  })
+
+  test(
+    'query monitor metrics with invalid step',
+    async () => {
+      const metrics = [
+        'cpuUsage',
+        'memoryUsage',
+        'storageUsage',
+        'databaseUsage',
+      ]
+
+      const query = {
+        q: metrics,
+        step: 50,
+      }
+
+      const res = await api.get(`/v1/monitor/${appid}/metrics`, {
+        params: query,
+        headers: { Authorization: `Bearer ${token}` },
+      })
+
+      expect(res.status).toBe(400)
+    },
+    10 * 1000,
+  )
+
+  test(
+    'query monitor metrics with invalid metrics',
+    async () => {
+      const metrics = ['cpuUsage', 'memoryUsage', 'storageUsage', 'invalid']
+
+      const query = {
+        q: metrics,
+        step: 50,
+      }
+
+      const res = await api.get(`/v1/monitor/${appid}/metrics`, {
+        params: query,
+        headers: { Authorization: `Bearer ${token}` },
+      })
+
+      expect(res.status).toBe(400)
+    },
+    10 * 1000,
+  )
+})

e2e/config.ts

@@ -1,4 +1,6 @@
+import * as dotenv from 'dotenv'
 
+dotenv.config()
 
 export class Config {
   static get DOMAIN() {
@@ -16,12 +18,17 @@ export class Config {
   }
 
   static get APISIX_ADMIN_URL() {
-    if (!process.env.APISIX_ADMIN_URL) throw new Error('APISIX_ADMIN_URL is not set')
-    return process.env.APISIX_ADMIN_URL || `http://${Config.DOMAIN}:9180/apisix/admin`
+    if (!process.env.APISIX_ADMIN_URL)
+      throw new Error('APISIX_ADMIN_URL is not set')
+    return (
+      process.env.APISIX_ADMIN_URL ||
+      `http://${Config.DOMAIN}:9180/apisix/admin`
+    )
   }
 
   static get APISIX_ADMIN_KEY() {
-    if (!process.env.APISIX_ADMIN_KEY) throw new Error('APISIX_ADMIN_KEY is not set')
+    if (!process.env.APISIX_ADMIN_KEY)
+      throw new Error('APISIX_ADMIN_KEY is not set')
     return process.env.APISIX_ADMIN_KEY
   }
 
@@ -36,4 +43,9 @@ export class Config {
   static get TEST_APP_NAME() {
     return process.env.TEST_APP_NAME || 'testing-e2e-application-name'
   }
-}
+
+  static get TEST_APPID() {
+    if (!process.env.TEST_APPID) throw new Error('TEST_APPID is not set')
+    return process.env.TEST_APPID
+  }
+}

e2e/package.json

@@ -16,6 +16,7 @@
   },
   "dependencies": {
     "axios": "^1.4.0",
+    "dotenv": "^16.3.1",
     "mongodb": "^5.7.0"
   }
 }

lerna.json

@@ -6,7 +6,8 @@
     "./web",
     "./runtimes/nodejs",
     "./cli",
-    "./services/*"
+    "./services/*",
+    "./e2e"
   ],
   "version": "1.0.0-beta.10",
   "command": {

server/src/app.module.ts

@@ -29,6 +29,7 @@ import { GroupModule } from './group/group.module'
 import { APP_INTERCEPTOR } from '@nestjs/core'
 import { AppInterceptor } from './app.interceptor'
 import { InterceptorModule } from './interceptor/interceptor.module'
+import { MonitorModule } from './monitor/monitor.module'
 
 @Module({
   imports: [
@@ -74,6 +75,7 @@ import { InterceptorModule } from './interceptor/interceptor.module'
     RecycleBinModule,
     GroupModule,
     InterceptorModule,
+    MonitorModule,
   ],
   controllers: [AppController],
   providers: [

server/src/constants.ts

@@ -10,6 +10,10 @@ export class ServerConfig {
     return process.env.DATABASE_URL
   }
 
+  static get PROMETHEUS_URL() {
+    return process.env.PROMETHEUS_URL
+  }
+
   static get METERING_DATABASE_URL() {
     if (!process.env.METERING_DATABASE_URL) {
       throw new Error('METERING_DATABASE_URL is not defined')
@@ -172,6 +176,10 @@ export class ServerConfig {
     return process.env.DEFAULT_REGION_LOG_SERVER_DATABASE_URL
   }
 
+  static get DEFAULT_REGION_PROMETHEUS_URL() {
+    return process.env.DEFAULT_REGION_PROMETHEUS_URL
+  }
+
   // HTTP interceptor
   static get HTTP_INTERCEPTOR_URL() {
     return process.env.HTTP_INTERCEPTOR_URL

server/src/initializer/initializer.service.ts

@@ -71,6 +71,9 @@ export class InitializerService {
         secret: ServerConfig.DEFAULT_REGION_LOG_SERVER_SECRET,
         databaseUrl: ServerConfig.DEFAULT_REGION_LOG_SERVER_DATABASE_URL,
       },
+      prometheusConf: {
+        apiUrl: ServerConfig.DEFAULT_REGION_PROMETHEUS_URL,
+      },
       updatedAt: new Date(),
       createdAt: new Date(),
       state: 'Active',

server/src/monitor/dto/query-metrics.dto.ts

@@ -0,0 +1,22 @@
+import { ApiProperty } from '@nestjs/swagger'
+import { IsArray, IsEnum, IsNumber, Max, Min } from 'class-validator'
+import { MonitorMetric } from '../monitor.service'
+import { Transform } from 'class-transformer'
+
+export class QueryMetricsDto {
+  @ApiProperty({ isArray: true, enum: MonitorMetric })
+  @IsEnum(MonitorMetric, { each: true })
+  @IsArray()
+  q: MonitorMetric[]
+
+  @ApiProperty({
+    minimum: 60,
+    maximum: 3600,
+    description: 'Query step in seconds',
+  })
+  @IsNumber()
+  @Min(60)
+  @Max(3600)
+  @Transform(({ value }) => Number(value))
+  step: number
+}