UBERF-8553: Stats as separate service

Signed-off-by: Andrey Sobolev <[email protected]>

.vscode/launch.json

@@ -94,6 +94,7 @@
         "REGION_INFO": "|Mongo;pg|Postgree",
         "ACCOUNT_PORT": "3000",
         "FRONT_URL": "http://localhost:8080",
+        "STATS_URL": "http://host.docker.internal:4900",
         "SES_URL": "",
         // "DB_NS": "account-2",
         // "WS_LIVENESS_DAYS": "1",
@@ -111,6 +112,22 @@
       "cwd": "${workspaceRoot}/pods/account",
       "protocol": "inspector"
     },
+    {
+      "name": "Debug Stats",
+      "type": "node",
+      "request": "launch",
+      "args": ["src/__start.ts"],
+      "env": {
+        "PORT": "4900",
+        "SERVER_SECRET": "secret",
+      },
+      "runtimeVersion": "20",
+      "runtimeArgs": ["--nolazy", "-r", "ts-node/register"],
+      "sourceMaps": true,
+      "outputCapture": "std",
+      "cwd": "${workspaceRoot}/pods/stats",
+      "protocol": "inspector"
+    },
     {
       "name": "Debug Workspace",
       "type": "node",

common/config/rush/command-line.json

@@ -238,7 +238,7 @@
       "summary": "Build docker with platform",
       "description": "use to build all docker containers required for platform",
       "safeForSimultaneousRushProcesses": true,
-      "shellCommand": "rush docker:build -p 20 --to @hcengineering/pod-server --to @hcengineering/pod-front --to @hcengineering/prod --to @hcengineering/pod-account --to @hcengineering/pod-workspace --to @hcengineering/pod-collaborator --to @hcengineering/tool --to @hcengineering/pod-print --to @hcengineering/pod-sign --to @hcengineering/pod-analytics-collector --to @hcengineering/rekoni-service --to @hcengineering/pod-ai-bot --to @hcengineering/import-tool"
+      "shellCommand": "rush docker:build -p 20 --to @hcengineering/pod-server --to @hcengineering/pod-front --to @hcengineering/prod --to @hcengineering/pod-account --to @hcengineering/pod-workspace --to @hcengineering/pod-collaborator --to @hcengineering/tool --to @hcengineering/pod-print --to @hcengineering/pod-sign --to @hcengineering/pod-analytics-collector --to @hcengineering/rekoni-service --to @hcengineering/pod-ai-bot --to @hcengineering/import-tool --to @hcengineering/pod-stats"
     },        
     {
       "commandKind": "global",

desktop/src/ui/platform.ts

@@ -208,6 +208,7 @@ export async function configurePlatform (): Promise<void> {
   setMetadata(presentation.metadata.PreviewConfig, parsePreviewConfig(config.PREVIEW_CONFIG))
   setMetadata(presentation.metadata.UploadConfig, parseUploadConfig(config.UPLOAD_CONFIG, config.UPLOAD_URL))
   setMetadata(presentation.metadata.FrontUrl, config.FRONT_URL)
+  setMetadata(presentation.metadata.StatsUrl, config.STATS_URL)
 
   setMetadata(textEditor.metadata.Collaborator, config.COLLABORATOR ?? '')
 

desktop/src/ui/types.ts

@@ -35,6 +35,8 @@ export interface Config {
   DESKTOP_UPDATES_URL?: string
   DESKTOP_UPDATES_CHANNEL?: string
   TELEGRAM_BOT_URL?: string
+
+  STATS_URL?: string
 }
 
 export interface Branding {

dev/docker-compose.yaml

@@ -66,13 +66,15 @@ services:
     links:
       - mongodb
       - minio
+      - stats
     ports:
       - 3000:3000
     volumes:
       - ./branding.json:/var/cfg/branding.json
     environment:
       - ACCOUNT_PORT=3000
       - SERVER_SECRET=secret
+      - STATS_URL=http://host.docker.internal:4900
       # - DB_URL=postgresql://postgres:example@postgres:5432
       - DB_URL=${MONGO_URL}
       # - DB_NS=account-2
@@ -91,6 +93,18 @@ services:
       # - INIT_SCRIPT_URL=https://raw.githubusercontent.com/hcengineering/init/main/script.yaml
       # - INIT_WORKSPACE=onboarding      
     restart: unless-stopped
+  stats:
+    image: hardcoreeng/stats
+    extra_hosts:
+      - "host.docker.internal:host-gateway"
+    ports:
+      - 4900:4900
+    volumes:
+      - ./branding.json:/var/cfg/branding.json
+    environment:
+      - PORT=4900
+      - SERVER_SECRET=secret
+    restart: unless-stopped    
   workspace:
     image: hardcoreeng/workspace
     extra_hosts:
@@ -101,12 +115,14 @@ services:
     links:
       - mongodb
       - minio
+      - stats
     volumes:
       - ./branding.json:/var/cfg/branding.json
     environment:
       # - WS_OPERATION=create
       - SERVER_SECRET=secret
       - DB_URL=${MONGO_URL}
+      - STATS_URL=http://host.docker.internal:4900
       # - DB_URL=postgresql://postgres:example@postgres:5432
       - SES_URL=
       - STORAGE_CONFIG=${STORAGE_CONFIG}
@@ -126,12 +142,14 @@ services:
     links:
       - postgres
       - minio
+      - stats
     volumes:
       - ./branding.json:/var/cfg/branding.json
     environment:
       # - WS_OPERATION=create
       - SERVER_SECRET=secret
       - DB_URL=postgresql://postgres:example@postgres:5432
+      - STATS_URL=http://host.docker.internal:4900
       - SES_URL=
       - REGION=pg
       - STORAGE_CONFIG=${STORAGE_CONFIG}
@@ -152,13 +170,15 @@ services:
       - mongodb
       - minio
       - transactor
+      - stats
     ports:
       - 3078:3078
     environment:
       - COLLABORATOR_PORT=3078
       - SECRET=secret
       - ACCOUNTS_URL=http://host.docker.internal:3000
       - STORAGE_CONFIG=${STORAGE_CONFIG}
+      - STATS_URL=http://host.docker.internal:4900
     restart: unless-stopped
   front:
     image: hardcoreeng/front
@@ -170,13 +190,15 @@ services:
       - elastic
       - transactor
       - collaborator
+      - stats
     ports:
       - 8087:8080
       - 8088:8080
     environment:
       - SERVER_PORT=8080
       - SERVER_SECRET=secret
       - ACCOUNTS_URL=http://host.docker.internal:3000
+      - STATS_URL=http://host.docker.internal:4900
       - UPLOAD_URL=/files
       - ELASTIC_URL=http://host.docker.internal:9200
       - GMAIL_URL=http://host.docker.internal:8088
@@ -204,6 +226,7 @@ services:
       - minio
       - rekoni
       - account
+      - stats
       # - apm-server
     ports:
       - 3333:3333
@@ -216,6 +239,7 @@ services:
       - SERVER_PORT=3333
       - SERVER_SECRET=secret
       - ENABLE_COMPRESSION=true
+      - STATS_URL=http://host.docker.internal:4900
       - ELASTIC_URL=http://host.docker.internal:9200
       # - DB_URL=postgresql://postgres:example@postgres:5432
       - DB_URL=${MONGO_URL}
@@ -244,6 +268,7 @@ services:
       - minio
       - rekoni
       - account
+      - stats
       # - apm-server
     ports:
       - 3331:3331
@@ -257,6 +282,7 @@ services:
       - SERVER_SECRET=secret
       - ENABLE_COMPRESSION=true
       - ELASTIC_URL=http://host.docker.internal:9200
+      - STATS_URL=http://host.docker.internal:4900
       - DB_URL=postgresql://postgres:example@postgres:5432
       - MONGO_URL=${MONGO_URL}
       - 'MONGO_OPTIONS={"appName": "transactor-pg", "maxPoolSize": 1}'
@@ -291,6 +317,7 @@ services:
     environment:
       - SECRET=secret
       - STORAGE_CONFIG=${STORAGE_CONFIG}
+      - STATS_URL=http://host.docker.internal:4900
     deploy:
       resources:
         limits:
@@ -314,6 +341,7 @@ services:
       - CERTIFICATE_PATH=/var/cfg/certificate.p12
       - SERVICE_ID=sign-service      
       - BRANDING_PATH=/var/cfg/branding.json
+      - STATS_URL=http://host.docker.internal:4900
     deploy:
       resources:
         limits:
@@ -333,6 +361,7 @@ services:
       - SERVICE_ID=analytics-collector-service
       - ACCOUNTS_URL=http://host.docker.internal:3000
       - SUPPORT_WORKSPACE=support
+      - STATS_URL=http://host.docker.internal:4900
     deploy:
       resources:
         limits:
@@ -354,6 +383,7 @@ services:
       - PASSWORD=password
       - AVATAR_PATH=./avatar.png
       - AVATAR_CONTENT_TYPE=.png
+      - STATS_URL=http://host.docker.internal:4900
     deploy:
       resources:
         limits:
@@ -372,6 +402,7 @@ services:
 #      - DOMAIN=domain
 #      - ACCOUNTS_URL=http://host.docker.internal:3000
 #      - SERVICE_ID=telegram-bot-service
+#      - STATS_URL=http://host.docker.internal:4900
 #    deploy:
 #      resources:
 #        limits:

dev/prod/public/config-dev.json

@@ -6,5 +6,6 @@
   "GMAIL_URL": "https://gmail.hc.engineering",
   "CALENDAR_URL": "https://calendar.hc.engineering",
   "REKONI_URL": "https://rekoni.hc.engineering",
-  "COLLABORATOR_URL": "wss://collaborator.hc.engineering"
+  "COLLABORATOR_URL": "wss://collaborator.hc.engineering",
+  "STATS_URL": "https://stats.hc.engineering"
 }

dev/prod/public/config-huly.json

@@ -19,5 +19,6 @@
   "SIGN_URL": "https://sign.huly.app",
   "PRINT_URL": "https://print.huly.app",
   "DESKTOP_UPDATES_CHANNEL": "huly",
-  "TELEGRAM_BOT_URL": "https://telegram-bot.huly.app"
+  "TELEGRAM_BOT_URL": "https://telegram-bot.huly.app",
+  "STATS_URL": "https://stats.huly.app"
 }

dev/prod/public/config.json

@@ -16,5 +16,6 @@
   "AI_URL": "http://localhost:4010",
   "BRANDING_URL": "/branding.json",
   "VERSION": null,
-  "MODEL_VERSION": null
+  "MODEL_VERSION": null,
+  "STATS_URL": "http://localhost:4900"
 }

dev/prod/src/platform.ts

@@ -156,6 +156,7 @@ export interface Config {
   FRONT_URL?: string
   PREVIEW_CONFIG?: string
   UPLOAD_CONFIG?: string
+  STATS_URL?: string
 }
 
 export interface Branding {
@@ -300,6 +301,7 @@ export async function configurePlatform() {
   setMetadata(presentation.metadata.FrontUrl, config.FRONT_URL)
   setMetadata(presentation.metadata.PreviewConfig, parsePreviewConfig(config.PREVIEW_CONFIG))
   setMetadata(presentation.metadata.UploadConfig, parseUploadConfig(config.UPLOAD_CONFIG, config.UPLOAD_URL))
+  setMetadata(presentation.metadata.StatsUrl, config.STATS_URL)
 
   setMetadata(textEditor.metadata.Collaborator, config.COLLABORATOR)
 

packages/presentation/src/index.ts

@@ -67,3 +67,4 @@ export * from './search'
 export * from './image'
 export * from './preview'
 export * from './sound'
+export * from './stats'

packages/presentation/src/plugin.ts

@@ -142,7 +142,8 @@ export default plugin(presentationId, {
     UploadConfig: '' as Metadata<UploadConfig>,
     PreviewConfig: '' as Metadata<PreviewConfig | undefined>,
     ClientHook: '' as Metadata<ClientHook>,
-    SessionId: '' as Metadata<string>
+    SessionId: '' as Metadata<string>,
+    StatsUrl: '' as Metadata<string>
   },
   status: {
     FileTooLarge: '' as StatusCode

packages/presentation/src/stats.ts

@@ -0,0 +1,58 @@
+import type { Metrics } from '@hcengineering/core'
+
+// Copy from server/core/stats.ts for UI usage.
+export interface MemoryStatistics {
+  memoryUsed: number
+  memoryTotal: number
+  memoryRSS: number
+  freeMem: number
+  totalMem: number
+}
+export interface CPUStatistics {
+  usage: number
+  cores: number
+}
+
+/**
+ * @public
+ */
+export interface StatisticsElement {
+  find: number
+  tx: number
+}
+
+export interface UserStatistics {
+  userId: string
+  sessionId: string
+  data: any
+  mins5: StatisticsElement
+  total: StatisticsElement
+  current: StatisticsElement
+}
+
+export interface WorkspaceStatistics {
+  sessions: UserStatistics[]
+  workspaceName: string
+  wsId: string
+  sessionsTotal: number
+  clientsTotal: number
+
+  service?: string
+}
+export interface ServiceStatistics {
+  serviceName: string // A service category
+  memory: MemoryStatistics
+  cpu: CPUStatistics
+  stats?: Metrics
+  workspaces?: WorkspaceStatistics[]
+}
+
+export interface OverviewStatistics {
+  memory: MemoryStatistics
+  cpu: CPUStatistics
+  data: Record<string, Omit<ServiceStatistics, 'stats' | 'workspaces'>>
+  usersTotal: number
+  connectionsTotal: number
+  admin: boolean
+  workspaces: WorkspaceStatistics[]
+}