UBERF-8525: Fix Github stuck on adding new integrations (#7009)

Signed-off-by: Andrey Sobolev <[email protected]>
hcengineering · Oct 22, 2024 · 418d810 · 418d810
1 parent 75c63a4
commit 418d810
Show file tree

Hide file tree

Showing 2 changed files with 41 additions and 13 deletions.
diff --git a/services/github/pod-github/src/platform.ts b/services/github/pod-github/src/platform.ts
@@ -162,10 +162,15 @@ export class PlatformWorker {
         errors = true
       }
       await new Promise<void>((resolve) => {
-        this.triggerCheckWorkspaces = resolve
-        this.ctx.info('Workspaces check triggered')
+        this.triggerCheckWorkspaces = () => {
+          this.ctx.info('Workspaces check triggered')
+          this.triggerCheckWorkspaces = () => {}
+          resolve()
+        }
         if (errors) {
-          setTimeout(resolve, 5000)
+          setTimeout(() => {
+            this.triggerCheckWorkspaces()
+          }, 5000)
         }
       })
     }
@@ -650,6 +655,9 @@ export class PlatformWorker {
   }
 
   private async checkWorkspaces (): Promise<boolean> {
+    this.ctx.info('************************* Check workspaces ************************* ', {
+      workspaces: this.clients.size
+    })
     let workspaces = await this.getWorkspaces()
     if (process.env.GITHUB_USE_WS !== undefined) {
       workspaces = [process.env.GITHUB_USE_WS]
@@ -660,7 +668,14 @@ export class PlatformWorker {
     let errors = 0
     let idx = 0
     const connecting = new Map<string, number>()
+    const st = Date.now()
     const connectingInfo = setInterval(() => {
+      this.ctx.info('****** connecting to workspaces ******', {
+        connecting: connecting.size,
+        time: Date.now() - st,
+        workspaces: workspaces.length,
+        queue: rateLimiter.processingQueue.size
+      })
       for (const [c, d] of connecting.entries()) {
         this.ctx.info('connecting to workspace', { workspace: c, time: Date.now() - d })
       }
@@ -727,7 +742,7 @@ export class PlatformWorker {
             }
           )
           if (worker !== undefined) {
-            workerCtx.info('Register worker Done', {
+            workerCtx.info('************************* Register worker Done ************************* ', {
               workspaceId: workspaceInfo.workspaceId,
               workspace: workspaceInfo.workspace,
               index: widx,
@@ -736,12 +751,15 @@ export class PlatformWorker {
             // No if no integration, we will try connect one more time in a time period
             this.clients.set(workspace, worker)
           } else {
-            workerCtx.info('Failed Register worker, timeout or integrations removed', {
-              workspaceId: workspaceInfo.workspaceId,
-              workspace: workspaceInfo.workspace,
-              index: widx,
-              total: workspaces.length
-            })
+            workerCtx.info(
+              '************************* Failed Register worker, timeout or integrations removed *************************',
+              {
+                workspaceId: workspaceInfo.workspaceId,
+                workspace: workspaceInfo.workspace,
+                index: widx,
+                total: workspaces.length
+              }
+            )
             errors++
           }
         } catch (e: any) {
@@ -754,21 +772,30 @@ export class PlatformWorker {
         }
       })
     }
+    this.ctx.info('************************* Waiting To complete Workspace processing ************************* ', {
+      workspaces: this.clients.size,
+      rateLimiter: rateLimiter.processingQueue.size
+    })
     try {
       await rateLimiter.waitProcessing()
     } catch (e: any) {
       Analytics.handleError(e)
       errors++
     }
     clearInterval(connectingInfo)
+
+    this.ctx.info('************************* Check close deleted ************************* ', {
+      workspaces: this.clients.size,
+      deleted: toDelete.size
+    })
     // Close deleted workspaces
     for (const deleted of Array.from(toDelete.keys())) {
       const ws = this.clients.get(deleted)
       if (ws !== undefined) {
         try {
           this.ctx.info('workspace removed from tracking list', { workspace: deleted })
           this.clients.delete(deleted)
-          await ws.close()
+          void ws.close()
         } catch (err: any) {
           Analytics.handleError(err)
           errors++

diff --git a/services/github/pod-github/src/worker.ts b/services/github/pod-github/src/worker.ts
@@ -127,8 +127,9 @@ export class GithubWorker implements IntegrationManager {
     this.closing = true
     this.ctx.warn('Closing', { workspace: this.workspace.name })
     this.triggerSync()
-    await this.syncPromise
-    this.ctx.warn('ClosingDone', { workspace: this.workspace.name })
+    await Promise.all([await this.syncPromise, new Promise<void>((resolve) => setTimeout(resolve, 5000))])
+
+    this.ctx.warn('Closing Done', { workspace: this.workspace.name })
     await this.client.close()
   }