From 86f039633b08a0a123d18eca49c28f28f3a5e3b2 Mon Sep 17 00:00:00 2001 From: Andrey Sobolev Date: Mon, 28 Oct 2024 19:17:37 +0700 Subject: [PATCH] UBERF-8553: Stats as separate service Signed-off-by: Andrey Sobolev --- .vscode/launch.json | 17 ++ common/config/rush/command-line.json | 2 +- common/config/rush/pnpm-lock.yaml | 43 ++++ desktop/src/ui/platform.ts | 1 + desktop/src/ui/types.ts | 2 + dev/docker-compose.yaml | 31 +++ dev/prod/public/config-dev.json | 3 +- dev/prod/public/config-huly.json | 3 +- dev/prod/public/config.json | 3 +- dev/prod/src/platform.ts | 2 + packages/presentation/src/index.ts | 1 + packages/presentation/src/plugin.ts | 3 +- packages/presentation/src/stats.ts | 58 +++++ .../src/components/MetricsStats.svelte | 38 +++ .../src/components/ServerManager.svelte | 29 +-- .../ServerManagerAccountStatistics.svelte | 65 ----- ...ServerManagerCollaboratorStatistics.svelte | 37 --- .../ServerManagerFrontStatistics.svelte | 35 --- .../ServerManagerServerStatistics.svelte | 106 ++++---- .../src/components/ServerManagerUsers.svelte | 223 +++++++--------- .../components/statistics/MetricsInfo.svelte | 37 +-- pods/account/src/__start.ts | 46 ++-- pods/backup/src/index.ts | 48 ++-- pods/collaborator/package.json | 3 +- pods/collaborator/src/__start.ts | 45 ++-- pods/front/src/__start.ts | 27 +- pods/server/src/__start.ts | 51 +++- pods/server/src/metrics.ts | 57 ----- pods/server/src/server.ts | 33 ++- pods/stats/.eslintrc.js | 7 + pods/stats/.npmignore | 4 + pods/stats/Dockerfile | 15 ++ pods/stats/build.sh | 20 ++ pods/stats/config/rig.json | 5 + pods/stats/jest.config.js | 7 + pods/stats/package.json | 65 +++++ pods/stats/src/stats.ts | 239 ++++++++++++++++++ pods/stats/tsconfig.json | 10 + pods/workspace/src/__start.ts | 54 ++-- rush.json | 5 + server/core/src/index.ts | 1 + server/core/src/stats.ts | 155 ++++++++++++ server/core/src/types.ts | 9 +- server/server/src/sessionManager.ts | 29 ++- server/ws/src/__tests__/server.test.ts | 4 +- services/ai-bot/pod-ai-bot/src/start.ts | 10 +- .../pod-analytics-collector/src/main.ts | 24 +- services/github/pod-github/src/index.ts | 41 ++- services/gmail/pod-gmail/src/main.ts | 5 +- services/love/src/main.ts | 8 +- services/print/pod-print/src/server.ts | 6 +- services/rekoni/package.json | 3 +- services/rekoni/src/server.ts | 4 +- services/sign/pod-sign/src/server.ts | 6 +- .../pod-telegram-bot/src/start.ts | 41 +-- services/telegram/pod-telegram/src/main.ts | 5 +- 56 files changed, 1157 insertions(+), 674 deletions(-) create mode 100644 packages/presentation/src/stats.ts create mode 100644 plugins/workbench-resources/src/components/MetricsStats.svelte delete mode 100644 plugins/workbench-resources/src/components/ServerManagerAccountStatistics.svelte delete mode 100644 plugins/workbench-resources/src/components/ServerManagerCollaboratorStatistics.svelte delete mode 100644 plugins/workbench-resources/src/components/ServerManagerFrontStatistics.svelte delete mode 100644 pods/server/src/metrics.ts create mode 100644 pods/stats/.eslintrc.js create mode 100644 pods/stats/.npmignore create mode 100644 pods/stats/Dockerfile create mode 100755 pods/stats/build.sh create mode 100644 pods/stats/config/rig.json create mode 100644 pods/stats/jest.config.js create mode 100644 pods/stats/package.json create mode 100644 pods/stats/src/stats.ts create mode 100644 pods/stats/tsconfig.json create mode 100644 server/core/src/stats.ts diff --git a/.vscode/launch.json b/.vscode/launch.json index b22678949b..59e9f2e294 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -94,6 +94,7 @@ "REGION_INFO": "|Mongo;pg|Postgree", "ACCOUNT_PORT": "3000", "FRONT_URL": "http://localhost:8080", + "STATS_URL": "http://host.docker.internal:4900", "SES_URL": "", // "DB_NS": "account-2", // "WS_LIVENESS_DAYS": "1", @@ -111,6 +112,22 @@ "cwd": "${workspaceRoot}/pods/account", "protocol": "inspector" }, + { + "name": "Debug Stats", + "type": "node", + "request": "launch", + "args": ["src/__start.ts"], + "env": { + "PORT": "4900", + "SERVER_SECRET": "secret", + }, + "runtimeVersion": "20", + "runtimeArgs": ["--nolazy", "-r", "ts-node/register"], + "sourceMaps": true, + "outputCapture": "std", + "cwd": "${workspaceRoot}/pods/stats", + "protocol": "inspector" + }, { "name": "Debug Workspace", "type": "node", diff --git a/common/config/rush/command-line.json b/common/config/rush/command-line.json index 21a9639e85..c6e75745b4 100644 --- a/common/config/rush/command-line.json +++ b/common/config/rush/command-line.json @@ -238,7 +238,7 @@ "summary": "Build docker with platform", "description": "use to build all docker containers required for platform", "safeForSimultaneousRushProcesses": true, - "shellCommand": "rush docker:build -p 20 --to @hcengineering/pod-server --to @hcengineering/pod-front --to @hcengineering/prod --to @hcengineering/pod-account --to @hcengineering/pod-workspace --to @hcengineering/pod-collaborator --to @hcengineering/tool --to @hcengineering/pod-print --to @hcengineering/pod-sign --to @hcengineering/pod-analytics-collector --to @hcengineering/rekoni-service --to @hcengineering/pod-ai-bot --to @hcengineering/import-tool" + "shellCommand": "rush docker:build -p 20 --to @hcengineering/pod-server --to @hcengineering/pod-front --to @hcengineering/prod --to @hcengineering/pod-account --to @hcengineering/pod-workspace --to @hcengineering/pod-collaborator --to @hcengineering/tool --to @hcengineering/pod-print --to @hcengineering/pod-sign --to @hcengineering/pod-analytics-collector --to @hcengineering/rekoni-service --to @hcengineering/pod-ai-bot --to @hcengineering/import-tool --to @hcengineering/pod-stats" }, { "commandKind": "global", diff --git a/common/config/rush/pnpm-lock.yaml b/common/config/rush/pnpm-lock.yaml index 4c8623f4f3..81ea23b318 100644 --- a/common/config/rush/pnpm-lock.yaml +++ b/common/config/rush/pnpm-lock.yaml @@ -632,6 +632,9 @@ dependencies: '@rush-temp/pod-sign': specifier: file:./projects/pod-sign.tgz version: file:projects/pod-sign.tgz + '@rush-temp/pod-stats': + specifier: file:./projects/pod-stats.tgz + version: file:projects/pod-stats.tgz '@rush-temp/pod-telegram': specifier: file:./projects/pod-telegram.tgz version: file:projects/pod-telegram.tgz(bufferutil@4.0.8)(ts-node@10.9.2)(utf-8-validate@6.0.4) @@ -31461,6 +31464,46 @@ packages: - supports-color dev: false + file:projects/pod-stats.tgz: + resolution: {integrity: sha512-vS1oRj3hDBzCAI0SagseVBqSDxd1puF9OQJRbCcn1V0MTYC5MwW8ndQGGG1W45jMK37vgHvgFveUn7FEEhIXOw==, tarball: file:projects/pod-stats.tgz} + name: '@rush-temp/pod-stats' + version: 0.0.0 + dependencies: + '@koa/cors': 5.0.0 + '@types/jest': 29.5.12 + '@types/koa': 2.15.0 + '@types/koa-bodyparser': 4.3.12 + '@types/koa-router': 7.4.8 + '@types/koa__cors': 5.0.0 + '@types/node': 20.11.19 + '@typescript-eslint/eslint-plugin': 6.21.0(@typescript-eslint/parser@6.21.0)(eslint@8.56.0)(typescript@5.6.2) + '@typescript-eslint/parser': 6.21.0(eslint@8.56.0)(typescript@5.6.2) + cross-env: 7.0.3 + esbuild: 0.20.1 + eslint: 8.56.0 + eslint-config-standard-with-typescript: 40.0.0(@typescript-eslint/eslint-plugin@6.21.0)(eslint-plugin-import@2.29.1)(eslint-plugin-n@15.7.0)(eslint-plugin-promise@6.1.1)(eslint@8.56.0)(typescript@5.6.2) + eslint-plugin-import: 2.29.1(eslint@8.56.0) + eslint-plugin-n: 15.7.0(eslint@8.56.0) + eslint-plugin-promise: 6.1.1(eslint@8.56.0) + jest: 29.7.0(@types/node@20.11.19)(ts-node@10.9.2) + koa: 2.15.3 + koa-bodyparser: 4.4.1 + koa-router: 12.0.1 + prettier: 3.2.5 + ts-jest: 29.1.2(esbuild@0.20.1)(jest@29.7.0)(typescript@5.6.2) + ts-node: 10.9.2(@types/node@20.11.19)(typescript@5.6.2) + typescript: 5.6.2 + transitivePeerDependencies: + - '@babel/core' + - '@jest/types' + - '@swc/core' + - '@swc/wasm' + - babel-jest + - babel-plugin-macros + - node-notifier + - supports-color + dev: false + file:projects/pod-telegram-bot.tgz(bufferutil@4.0.8)(utf-8-validate@6.0.4): resolution: {integrity: sha512-Bq0gNcjaTU0PEYhroGlOPGUD1UsNhYswf32XQ5fgz//09GK9gEBx6qBknRNYGvbN5558bUyyzKyp0KsWVfySkQ==, tarball: file:projects/pod-telegram-bot.tgz} id: file:projects/pod-telegram-bot.tgz diff --git a/desktop/src/ui/platform.ts b/desktop/src/ui/platform.ts index 3a9585634b..74aecd76d9 100644 --- a/desktop/src/ui/platform.ts +++ b/desktop/src/ui/platform.ts @@ -208,6 +208,7 @@ export async function configurePlatform (): Promise { setMetadata(presentation.metadata.PreviewConfig, parsePreviewConfig(config.PREVIEW_CONFIG)) setMetadata(presentation.metadata.UploadConfig, parseUploadConfig(config.UPLOAD_CONFIG, config.UPLOAD_URL)) setMetadata(presentation.metadata.FrontUrl, config.FRONT_URL) + setMetadata(presentation.metadata.StatsUrl, config.STATS_URL) setMetadata(textEditor.metadata.Collaborator, config.COLLABORATOR ?? '') diff --git a/desktop/src/ui/types.ts b/desktop/src/ui/types.ts index 964b189a8b..14aca214cb 100644 --- a/desktop/src/ui/types.ts +++ b/desktop/src/ui/types.ts @@ -35,6 +35,8 @@ export interface Config { DESKTOP_UPDATES_URL?: string DESKTOP_UPDATES_CHANNEL?: string TELEGRAM_BOT_URL?: string + + STATS_URL?: string } export interface Branding { diff --git a/dev/docker-compose.yaml b/dev/docker-compose.yaml index 66198922a1..19160c3415 100644 --- a/dev/docker-compose.yaml +++ b/dev/docker-compose.yaml @@ -66,6 +66,7 @@ services: links: - mongodb - minio + - stats ports: - 3000:3000 volumes: @@ -73,6 +74,7 @@ services: environment: - ACCOUNT_PORT=3000 - SERVER_SECRET=secret + - STATS_URL=http://host.docker.internal:4900 # - DB_URL=postgresql://postgres:example@postgres:5432 - DB_URL=${MONGO_URL} # - DB_NS=account-2 @@ -91,6 +93,18 @@ services: # - INIT_SCRIPT_URL=https://raw.githubusercontent.com/hcengineering/init/main/script.yaml # - INIT_WORKSPACE=onboarding restart: unless-stopped + stats: + image: hardcoreeng/stats + extra_hosts: + - "host.docker.internal:host-gateway" + ports: + - 4900:4900 + volumes: + - ./branding.json:/var/cfg/branding.json + environment: + - PORT=4900 + - SERVER_SECRET=secret + restart: unless-stopped workspace: image: hardcoreeng/workspace extra_hosts: @@ -101,12 +115,14 @@ services: links: - mongodb - minio + - stats volumes: - ./branding.json:/var/cfg/branding.json environment: # - WS_OPERATION=create - SERVER_SECRET=secret - DB_URL=${MONGO_URL} + - STATS_URL=http://host.docker.internal:4900 # - DB_URL=postgresql://postgres:example@postgres:5432 - SES_URL= - STORAGE_CONFIG=${STORAGE_CONFIG} @@ -126,12 +142,14 @@ services: links: - postgres - minio + - stats volumes: - ./branding.json:/var/cfg/branding.json environment: # - WS_OPERATION=create - SERVER_SECRET=secret - DB_URL=postgresql://postgres:example@postgres:5432 + - STATS_URL=http://host.docker.internal:4900 - SES_URL= - REGION=pg - STORAGE_CONFIG=${STORAGE_CONFIG} @@ -152,6 +170,7 @@ services: - mongodb - minio - transactor + - stats ports: - 3078:3078 environment: @@ -159,6 +178,7 @@ services: - SECRET=secret - ACCOUNTS_URL=http://host.docker.internal:3000 - STORAGE_CONFIG=${STORAGE_CONFIG} + - STATS_URL=http://host.docker.internal:4900 restart: unless-stopped front: image: hardcoreeng/front @@ -170,6 +190,7 @@ services: - elastic - transactor - collaborator + - stats ports: - 8087:8080 - 8088:8080 @@ -177,6 +198,7 @@ services: - SERVER_PORT=8080 - SERVER_SECRET=secret - ACCOUNTS_URL=http://host.docker.internal:3000 + - STATS_URL=http://host.docker.internal:4900 - UPLOAD_URL=/files - ELASTIC_URL=http://host.docker.internal:9200 - GMAIL_URL=http://host.docker.internal:8088 @@ -204,6 +226,7 @@ services: - minio - rekoni - account + - stats # - apm-server ports: - 3333:3333 @@ -216,6 +239,7 @@ services: - SERVER_PORT=3333 - SERVER_SECRET=secret - ENABLE_COMPRESSION=true + - STATS_URL=http://host.docker.internal:4900 - ELASTIC_URL=http://host.docker.internal:9200 # - DB_URL=postgresql://postgres:example@postgres:5432 - DB_URL=${MONGO_URL} @@ -244,6 +268,7 @@ services: - minio - rekoni - account + - stats # - apm-server ports: - 3331:3331 @@ -257,6 +282,7 @@ services: - SERVER_SECRET=secret - ENABLE_COMPRESSION=true - ELASTIC_URL=http://host.docker.internal:9200 + - STATS_URL=http://host.docker.internal:4900 - DB_URL=postgresql://postgres:example@postgres:5432 - MONGO_URL=${MONGO_URL} - 'MONGO_OPTIONS={"appName": "transactor-pg", "maxPoolSize": 1}' @@ -291,6 +317,7 @@ services: environment: - SECRET=secret - STORAGE_CONFIG=${STORAGE_CONFIG} + - STATS_URL=http://host.docker.internal:4900 deploy: resources: limits: @@ -314,6 +341,7 @@ services: - CERTIFICATE_PATH=/var/cfg/certificate.p12 - SERVICE_ID=sign-service - BRANDING_PATH=/var/cfg/branding.json + - STATS_URL=http://host.docker.internal:4900 deploy: resources: limits: @@ -333,6 +361,7 @@ services: - SERVICE_ID=analytics-collector-service - ACCOUNTS_URL=http://host.docker.internal:3000 - SUPPORT_WORKSPACE=support + - STATS_URL=http://host.docker.internal:4900 deploy: resources: limits: @@ -354,6 +383,7 @@ services: - PASSWORD=password - AVATAR_PATH=./avatar.png - AVATAR_CONTENT_TYPE=.png + - STATS_URL=http://host.docker.internal:4900 deploy: resources: limits: @@ -372,6 +402,7 @@ services: # - DOMAIN=domain # - ACCOUNTS_URL=http://host.docker.internal:3000 # - SERVICE_ID=telegram-bot-service +# - STATS_URL=http://host.docker.internal:4900 # deploy: # resources: # limits: diff --git a/dev/prod/public/config-dev.json b/dev/prod/public/config-dev.json index 842ed6d3b8..8f0b1e22f2 100644 --- a/dev/prod/public/config-dev.json +++ b/dev/prod/public/config-dev.json @@ -6,5 +6,6 @@ "GMAIL_URL": "https://gmail.hc.engineering", "CALENDAR_URL": "https://calendar.hc.engineering", "REKONI_URL": "https://rekoni.hc.engineering", - "COLLABORATOR_URL": "wss://collaborator.hc.engineering" + "COLLABORATOR_URL": "wss://collaborator.hc.engineering", + "STATS_URL": "https://stats.hc.engineering" } \ No newline at end of file diff --git a/dev/prod/public/config-huly.json b/dev/prod/public/config-huly.json index b98b428b93..a7661a88e6 100644 --- a/dev/prod/public/config-huly.json +++ b/dev/prod/public/config-huly.json @@ -19,5 +19,6 @@ "SIGN_URL": "https://sign.huly.app", "PRINT_URL": "https://print.huly.app", "DESKTOP_UPDATES_CHANNEL": "huly", - "TELEGRAM_BOT_URL": "https://telegram-bot.huly.app" + "TELEGRAM_BOT_URL": "https://telegram-bot.huly.app", + "STATS_URL": "https://stats.huly.app" } \ No newline at end of file diff --git a/dev/prod/public/config.json b/dev/prod/public/config.json index f78cb037f6..d2f1c4d3a0 100644 --- a/dev/prod/public/config.json +++ b/dev/prod/public/config.json @@ -16,5 +16,6 @@ "AI_URL": "http://localhost:4010", "BRANDING_URL": "/branding.json", "VERSION": null, - "MODEL_VERSION": null + "MODEL_VERSION": null, + "STATS_URL": "http://localhost:4900" } \ No newline at end of file diff --git a/dev/prod/src/platform.ts b/dev/prod/src/platform.ts index 31e3752476..31c589860e 100644 --- a/dev/prod/src/platform.ts +++ b/dev/prod/src/platform.ts @@ -156,6 +156,7 @@ export interface Config { FRONT_URL?: string PREVIEW_CONFIG?: string UPLOAD_CONFIG?: string + STATS_URL?: string } export interface Branding { @@ -300,6 +301,7 @@ export async function configurePlatform() { setMetadata(presentation.metadata.FrontUrl, config.FRONT_URL) setMetadata(presentation.metadata.PreviewConfig, parsePreviewConfig(config.PREVIEW_CONFIG)) setMetadata(presentation.metadata.UploadConfig, parseUploadConfig(config.UPLOAD_CONFIG, config.UPLOAD_URL)) + setMetadata(presentation.metadata.StatsUrl, config.STATS_URL) setMetadata(textEditor.metadata.Collaborator, config.COLLABORATOR) diff --git a/packages/presentation/src/index.ts b/packages/presentation/src/index.ts index 9306e3e849..c43c424b20 100644 --- a/packages/presentation/src/index.ts +++ b/packages/presentation/src/index.ts @@ -67,3 +67,4 @@ export * from './search' export * from './image' export * from './preview' export * from './sound' +export * from './stats' diff --git a/packages/presentation/src/plugin.ts b/packages/presentation/src/plugin.ts index 0f826224c4..f47ef1bb7b 100644 --- a/packages/presentation/src/plugin.ts +++ b/packages/presentation/src/plugin.ts @@ -142,7 +142,8 @@ export default plugin(presentationId, { UploadConfig: '' as Metadata, PreviewConfig: '' as Metadata, ClientHook: '' as Metadata, - SessionId: '' as Metadata + SessionId: '' as Metadata, + StatsUrl: '' as Metadata }, status: { FileTooLarge: '' as StatusCode diff --git a/packages/presentation/src/stats.ts b/packages/presentation/src/stats.ts new file mode 100644 index 0000000000..a380c20d2e --- /dev/null +++ b/packages/presentation/src/stats.ts @@ -0,0 +1,58 @@ +import type { Metrics } from '@hcengineering/core' + +// Copy from server/core/stats.ts for UI usage. +export interface MemoryStatistics { + memoryUsed: number + memoryTotal: number + memoryRSS: number + freeMem: number + totalMem: number +} +export interface CPUStatistics { + usage: number + cores: number +} + +/** + * @public + */ +export interface StatisticsElement { + find: number + tx: number +} + +export interface UserStatistics { + userId: string + sessionId: string + data: any + mins5: StatisticsElement + total: StatisticsElement + current: StatisticsElement +} + +export interface WorkspaceStatistics { + sessions: UserStatistics[] + workspaceName: string + wsId: string + sessionsTotal: number + clientsTotal: number + + service?: string +} +export interface ServiceStatistics { + serviceName: string // A service category + memory: MemoryStatistics + cpu: CPUStatistics + stats?: Metrics + workspaces?: WorkspaceStatistics[] +} + +export interface OverviewStatistics { + memory: MemoryStatistics + cpu: CPUStatistics + data: Record> + usersTotal: number + connectionsTotal: number + admin: boolean + workspaces: WorkspaceStatistics[] +} diff --git a/plugins/workbench-resources/src/components/MetricsStats.svelte b/plugins/workbench-resources/src/components/MetricsStats.svelte new file mode 100644 index 0000000000..6cd1d15cd2 --- /dev/null +++ b/plugins/workbench-resources/src/components/MetricsStats.svelte @@ -0,0 +1,38 @@ + + +
+ {#if metricsData !== undefined} + + {/if} +
+ + diff --git a/plugins/workbench-resources/src/components/ServerManager.svelte b/plugins/workbench-resources/src/components/ServerManager.svelte index 80700cbc5c..99769ad5dc 100644 --- a/plugins/workbench-resources/src/components/ServerManager.svelte +++ b/plugins/workbench-resources/src/components/ServerManager.svelte @@ -1,8 +1,4 @@ - -{#if data} -
- - Mem: {data?.statistics?.memoryUsed} / {data?.statistics?.memoryTotal} CPU: {data?.statistics?.cpuUsage} - -
-{/if} - -{#if admin} -
-
-
1.
-
-
-{/if} -
- {#if metricsData !== undefined} - - {/if} -
- - diff --git a/plugins/workbench-resources/src/components/ServerManagerCollaboratorStatistics.svelte b/plugins/workbench-resources/src/components/ServerManagerCollaboratorStatistics.svelte deleted file mode 100644 index 0c45c272f3..0000000000 --- a/plugins/workbench-resources/src/components/ServerManagerCollaboratorStatistics.svelte +++ /dev/null @@ -1,37 +0,0 @@ - - -
- {#if metricsDataCollab !== undefined} - - {/if} -
- - diff --git a/plugins/workbench-resources/src/components/ServerManagerFrontStatistics.svelte b/plugins/workbench-resources/src/components/ServerManagerFrontStatistics.svelte deleted file mode 100644 index 015fc02d79..0000000000 --- a/plugins/workbench-resources/src/components/ServerManagerFrontStatistics.svelte +++ /dev/null @@ -1,35 +0,0 @@ - - -
- {#if metricsDataFront !== undefined} - - {/if} -
- - diff --git a/plugins/workbench-resources/src/components/ServerManagerServerStatistics.svelte b/plugins/workbench-resources/src/components/ServerManagerServerStatistics.svelte index 548596199b..841912306f 100644 --- a/plugins/workbench-resources/src/components/ServerManagerServerStatistics.svelte +++ b/plugins/workbench-resources/src/components/ServerManagerServerStatistics.svelte @@ -1,21 +1,17 @@ {#if data}
- Mem: {data.statistics.memoryUsed} / {data.statistics.memoryTotal} CPU: {data.statistics.cpuUsage} - - - TotalFind: {totalStats.find} / Total Tx: {totalStats.tx} + Connections: {data.connectionsTotal} + Users: {data.usersTotal} +
{/if} @@ -96,11 +61,38 @@ {/if} -
- {#if metricsData !== undefined} - - {/if} -
+{#if data} +
+ +
+
+ {#each Object.entries(data.data).sort((a, b) => a[1].serviceName.localeCompare(b[1].serviceName)) as kv} + + +
+ {kv[1].serviceName} - {kv[0]} +
+ + +
+ + {kv[1].memory.memoryUsed}/{kv[1].memory.memoryTotal} Mb + + + {kv[1].memory.memoryRSS} Mb + + +
+ {kv[1].cpu.usage}% +
+ +
+ + + + {/each} +
+{/if}