UBERF-8469: Fix exit from github service (#6921)

Signed-off-by: Andrey Sobolev <[email protected]>

services/github/pod-github/package.json

@@ -17,12 +17,12 @@
     "_phase:bundle": "rushx bundle",
     "_phase:docker-build": "rushx docker:build",
     "_phase:docker-staging": "rushx docker:staging",
-    "bundle": "mkdir -p bundle && esbuild src/index.ts --bundle --platform=node --outfile=bundle/bundle.js --log-level=error --sourcemap=external",
+    "bundle": "mkdir -p bundle && esbuild src/index.ts --keep-names --bundle --platform=node --outfile=bundle/bundle.js --log-level=error --sourcemap=external",
     "docker:build": "../../../common/scripts/docker_build.sh hardcoreeng/github",
     "docker:staging": "../../../common/scripts/docker_tag.sh hardcoreeng/github staging",
     "docker:push": "../../../common/scripts/docker_tag.sh hardcoreeng/github",
     "docker:tbuild": "rush bundle --to @hcengineering/pod-github && docker build -t hardcoreeng/github . --platform=linux/amd64 && ../../../common/scripts/docker_tag_push.sh hardcoreeng/github",
-    "run-local": "cross-env APP_ID=$(cat ../../../../uberflow_private/appid) PRIVATE_KEY=\"$(cat ../../../../uberflow_private/private-key.pem)\" CLIENT_ID=$(cat ../../../../uberflow_private/client-id) CLIENT_SECRET=$(cat ../../../../uberflow_private/client-secret) SERVER_SECRET=secret ACCOUNTS_URL=http://localhost:3000/ COLLABORATOR_URL=http://localhost:3078 MINIO_ACCESS_KEY=minioadmin MINIO_SECRET_KEY=minioadmin MINIO_ENDPOINT=localhost ts-node src/index.ts",
+    "run-local": "ruh.sh",
     "format": "format src",
     "_phase:build": "compile transpile src",
     "_phase:test": "jest --passWithNoTests --silent",

services/github/pod-github/run.sh

@@ -0,0 +1,13 @@
+export APP_ID="$POD_GITHUB_APPID"
+export CLIENT_ID="$POD_GITHUB_CLIENTID"
+export CLIENT_SECRET="$POD_GITHUB_CLIENT_SECRET"
+export PRIVATE_KEY="$POD_GITHUB_PRIVATE_KEY"
+export SERVER_SECRET=secret
+export ACCOUNTS_URL=http://localhost:3000
+export COLLABORATOR_URL=http://localhost:3078
+export MINIO_ACCESS_KEY=minioadminchmo
+export MINIO_SECRET_KEY=minioadmin
+export MINIO_ENDPOINT=localhost
+export MONGO_URL=mongodb://localhost:27017
+rush bundle --to @hcengineering/pod-github
+node $@ bundle/bundle.js

services/github/pod-github/src/index.ts

@@ -2,14 +2,14 @@
 // Copyright © 2023 Hardcore Engineering Inc.
 //
 
-import { MeasureMetricsContext, metricsToString, newMetrics } from '@hcengineering/core'
+import { Analytics } from '@hcengineering/analytics'
 import { SplitLogger, configureAnalytics } from '@hcengineering/analytics-service'
+import { MeasureMetricsContext, metricsToString, newMetrics } from '@hcengineering/core'
+import { loadBrandingMap } from '@hcengineering/server-core'
 import { writeFile } from 'fs/promises'
 import { join } from 'path'
 import config from './config'
 import { start } from './server'
-import { Analytics } from '@hcengineering/analytics'
-import { loadBrandingMap } from '@hcengineering/server-core'
 
 // Load and inc startID, to have easy logs.
 
@@ -39,11 +39,18 @@ const intTimer = setInterval(() => {
   }
 }, 30000)
 
-void start(metricsContext, loadBrandingMap(config.BrandingPath))
+let doOnClose: () => Promise<void> = async () => {}
+
+void start(metricsContext, loadBrandingMap(config.BrandingPath)).then((r) => {
+  doOnClose = r
+})
 
 const onClose = (): void => {
   clearInterval(intTimer)
   metricsContext.info('Closed')
+  void doOnClose().then((r) => {
+    process.exit(0)
+  })
 }
 
 process.on('uncaughtException', (e) => {

services/github/pod-github/src/platform.ts

@@ -96,6 +96,7 @@ export class PlatformWorker {
 
   async close (): Promise<void> {
     this.canceled = true
+    clearInterval(this.periodicSyncInterval)
     await Promise.all(
       [...this.clients.values()].map(async (worker) => {
         await worker.close()

services/github/pod-github/src/server.ts

@@ -18,7 +18,7 @@ import { decodeToken } from '@hcengineering/server-token'
 /**
  * @public
  */
-export async function start (ctx: MeasureContext, brandingMap: BrandingMap): Promise<void> {
+export async function start (ctx: MeasureContext, brandingMap: BrandingMap): Promise<() => Promise<void>> {
   // Create an authenticated Octokit client authenticated as a GitHub App
   ctx.info('Running Huly Github integration', { appId: config.AppID, clientID: config.ClientID })
 
@@ -184,8 +184,13 @@ export async function start (ctx: MeasureContext, brandingMap: BrandingMap): Pro
     }
   })
 
-  app.listen(port, () => {
+  const server = app.listen(port, () => {
     ctx.info(`Server is listening for events at: ${localWebhookUrl}`)
     ctx.info('Press Ctrl + C to quit.')
   })
+
+  return async () => {
+    await worker.close()
+    server.close()
+  }
 }

services/github/pod-github/src/sync/comments.ts

@@ -457,6 +457,9 @@ export class CommentSyncManager implements DocSyncManager {
     repositories: GithubIntegrationRepository[]
   ): Promise<void> {
     for (const repo of repositories) {
+      if (this.provider.isClosing()) {
+        break
+      }
       const syncKey = `${repo._id}:comment`
       if (repo.githubProject === undefined || !repo.enabled || integration.synchronized.has(syncKey)) {
         if (!repo.enabled) {
@@ -487,6 +490,9 @@ export class CommentSyncManager implements DocSyncManager {
       })
       try {
         for await (const data of i) {
+          if (this.provider.isClosing()) {
+            break
+          }
           const comments: CommentExternalData[] = data.data as any
           this.ctx.info('retrieve comments for', {
             repo: repo.name,

services/github/pod-github/src/sync/issueBase.ts

@@ -815,7 +815,7 @@ export abstract class IssueSyncManagerBase {
 
       // Collect field update.
       for (const [k, v] of Object.entries(platformUpdate)) {
-        const mapping = target.mappings.find((it) => it.name === k)
+        const mapping = target.mappings.filter((it) => it != null).find((it) => it.name === k)
         if (mapping === undefined) {
           continue
         }

services/github/pod-github/src/sync/issues.ts

@@ -985,6 +985,9 @@ export class IssueSyncManager extends IssueSyncManagerBase implements DocSyncMan
     let partsize = 50
     try {
       while (true) {
+        if (this.provider.isClosing()) {
+          break
+        }
         const idsPart = ids.splice(0, partsize)
         if (idsPart.length === 0) {
           break
@@ -1080,6 +1083,9 @@ export class IssueSyncManager extends IssueSyncManagerBase implements DocSyncMan
     repositories: GithubIntegrationRepository[]
   ): Promise<void> {
     for (const repo of repositories) {
+      if (this.provider.isClosing()) {
+        break
+      }
       const prj = projects.find((it) => repo.githubProject === it._id)
       if (prj === undefined) {
         continue
@@ -1128,6 +1134,9 @@ export class IssueSyncManager extends IssueSyncManagerBase implements DocSyncMan
       )
       try {
         for await (const data of i) {
+          if (this.provider.isClosing()) {
+            break
+          }
           const issues: IssueExternalData[] = data.repository.issues.nodes
           if (issues.some((issue) => issue.url === undefined && Object.keys(issue).length === 0)) {
             this.ctx.error('empty document content', {

services/github/pod-github/src/sync/projects.ts

@@ -376,6 +376,9 @@ export class ProjectsSyncManager implements DocSyncManager {
     repositories: GithubIntegrationRepository[]
   ): Promise<void> {
     for (const prj of projects) {
+      if (this.provider.isClosing()) {
+        break
+      }
       // Wait global project sync
       await integration.syncLock.get(prj._id)
 
@@ -449,6 +452,9 @@ export class ProjectsSyncManager implements DocSyncManager {
           (it) => it.space === prj._id
         )
         for (const m of milestones) {
+          if (this.provider.isClosing()) {
+            break
+          }
           try {
             let { projectStructure, wasUpdates } = await this.ctx.withLog(
               'update project structure',

services/github/pod-github/src/sync/pullrequests.ts

@@ -1433,6 +1433,9 @@ export class PullRequestSyncManager extends IssueSyncManagerBase implements DocS
     repositories: GithubIntegrationRepository[]
   ): Promise<void> {
     for (const repo of repositories) {
+      if (this.provider.isClosing()) {
+        break
+      }
       const prj = projects.find((it) => repo.githubProject === it._id)
       if (prj === undefined) {
         continue
@@ -1518,6 +1521,9 @@ export class PullRequestSyncManager extends IssueSyncManagerBase implements DocS
         }
       )
       for await (const data of pullRequestIterator) {
+        if (this.provider.isClosing()) {
+          break
+        }
         const issues: PullRequestExternalData[] = data.repository.pullRequests.nodes
         this.ctx.info('retrieve pull requests for', {
           repo: repo.name,

services/github/pod-github/src/sync/users.ts

@@ -60,6 +60,9 @@ export class UsersSyncManager implements DocSyncManager {
     repositories: GithubIntegrationRepository[]
   ): Promise<void> {
     for (const repo of repositories) {
+      if (this.provider.isClosing()) {
+        break
+      }
       const syncKey = `${repo._id}:users`
       if (
         repo.githubProject === undefined ||
@@ -107,6 +110,9 @@ export class UsersSyncManager implements DocSyncManager {
     )
     try {
       for await (const data of assignableUsersIterator) {
+        if (this.provider.isClosing()) {
+          break
+        }
         const users: UserInfo[] = data.repository[key]?.nodes ?? []
         for (const d of users) {
           if (d.login !== undefined) {

services/github/pod-github/src/types.ts

@@ -129,6 +129,8 @@ export interface IntegrationManager {
   getProjectRepositories: (space: Ref<Space>) => Promise<GithubIntegrationRepository[]>
 
   getRepositoryById: (ref?: Ref<GithubIntegrationRepository> | null) => Promise<GithubIntegrationRepository | undefined>
+
+  isClosing: () => boolean
 }
 
 export type ExternalSyncField = 'externalVersion' | 'derivedVersion'

services/github/pod-github/src/worker.ts

@@ -117,11 +117,18 @@ export class GithubWorker implements IntegrationManager {
 
   personMapper: UsersSyncManager
 
+  isClosing (): boolean {
+    return this.closing
+  }
+
   async close (): Promise<void> {
     clearInterval(this.periodicTimer)
 
     this.closing = true
+    this.ctx.warn('Closing', { workspace: this.workspace.name })
+    this.triggerSync()
     await this.syncPromise
+    this.ctx.warn('ClosingDone', { workspace: this.workspace.name })
     await this.client.close()
   }
 
@@ -1347,6 +1354,9 @@ export class GithubWorker implements IntegrationManager {
   }
 
   private async waitChanges (): Promise<void> {
+    if (this.closing) {
+      return
+    }
     if (this.triggerRequests > 0 || this.updateRequests > 0) {
       this.ctx.info('Trigger check pending:', {
         requests: this.triggerRequests,
@@ -1404,6 +1414,9 @@ export class GithubWorker implements IntegrationManager {
   async _performFullSync (): Promise<void> {
     // Wait previous active sync
     for (const integration of this.integrations.values()) {
+      if (this.closing) {
+        break
+      }
       await this.ctx.withLog(
         'external sync',
         { installation: integration.installationName, workspace: this.workspace.name },
@@ -1442,6 +1455,9 @@ export class GithubWorker implements IntegrationManager {
           // Cleanup broken synchronized documents
 
           while (true) {
+            if (this.closing) {
+              break
+            }
             const withError = await derivedClient.findAll<any>(
               github.class.DocSyncInfo,
               { error: { $ne: null }, url: null },
@@ -1458,6 +1474,9 @@ export class GithubWorker implements IntegrationManager {
           }
 
           for (const { _class, mapper } of this.mappers) {
+            if (this.closing) {
+              break
+            }
             await this.ctx.withLog(
               'external sync',
               { _class: _class.join(', '), workspace: this.workspace.name },