fix: add correct indexes for get group key runs, improve queries (#786)

* fix: add correct indexes for get group key runs * chore: generate * fix: hash
hatchet-dev · Aug 20, 2024 · 67357cf · 67357cf
1 parent 081c37a
commit 67357cf
Show file tree

Hide file tree

Showing 11 changed files with 80 additions and 19 deletions.
diff --git a/pkg/repository/prisma/db/db_gen.go b/pkg/repository/prisma/db/db_gen.go
@@ -1095,6 +1095,13 @@ model GetGroupKeyRun {
   cancelledError String?
 
   @@index([deletedAt])
+  @@index([tenantId])
+  @@index([workerId])
+  @@index([createdAt])
+  // index for ListStepRunsToReassign, ListStepRunsToRequeue
+  @@index([tenantId, deletedAt, status])
+  // index for PollGetGroupKeyRuns
+  @@index([status, deletedAt, timeoutAt])
 }
 
 model WorkflowRunTriggeredBy {

diff --git a/pkg/repository/prisma/dbsqlc/get_group_key_runs.sql b/pkg/repository/prisma/dbsqlc/get_group_key_runs.sql
@@ -84,13 +84,8 @@ group_key_runs AS (
     WHERE
         ggr."tenantId" = @tenantId::uuid
         AND ggr."deletedAt" IS NULL
-        AND ((
-            ggr."status" = 'RUNNING'
-            AND w."lastHeartbeatAt" < NOW() - INTERVAL '30 seconds'
-        ) OR (
-            ggr."status" = 'ASSIGNED'
-            AND w."lastHeartbeatAt" < NOW() - INTERVAL '30 seconds'
-        ))
+        AND ggr."status" = ANY(ARRAY['RUNNING', 'ASSIGNED']::"StepRunStatus"[])
+        AND w."lastHeartbeatAt" < NOW() - INTERVAL '30 seconds'
     ORDER BY
         ggr."createdAt" ASC
     LIMIT
@@ -147,7 +142,7 @@ group_key_runs AS (
         ggr."tenantId" = @tenantId::uuid
         AND ggr."deletedAt" IS NULL
         AND ggr."requeueAfter" < NOW()
-        AND (ggr."status" = 'PENDING' OR ggr."status" = 'PENDING_ASSIGNMENT')
+        AND ggr."status" = ANY(ARRAY['PENDING', 'PENDING_ASSIGNMENT']::"StepRunStatus"[])
     ORDER BY
         ggr."createdAt" ASC
     LIMIT

diff --git a/pkg/repository/prisma/dbsqlc/get_group_key_runs.sql.go b/pkg/repository/prisma/dbsqlc/get_group_key_runs.sql.go
diff --git a/pkg/repository/prisma/dbsqlc/schema.sql b/pkg/repository/prisma/dbsqlc/schema.sql
@@ -880,12 +880,27 @@ CREATE INDEX "Event_tenantId_createdAt_idx" ON "Event"("tenantId" ASC, "createdA
 -- CreateIndex
 CREATE INDEX "Event_tenantId_idx" ON "Event"("tenantId" ASC);
 
+-- CreateIndex
+CREATE INDEX "GetGroupKeyRun_createdAt_idx" ON "GetGroupKeyRun"("createdAt" ASC);
+
 -- CreateIndex
 CREATE INDEX "GetGroupKeyRun_deletedAt_idx" ON "GetGroupKeyRun"("deletedAt" ASC);
 
 -- CreateIndex
 CREATE UNIQUE INDEX "GetGroupKeyRun_id_key" ON "GetGroupKeyRun"("id" ASC);
 
+-- CreateIndex
+CREATE INDEX "GetGroupKeyRun_status_deletedAt_timeoutAt_idx" ON "GetGroupKeyRun"("status" ASC, "deletedAt" ASC, "timeoutAt" ASC);
+
+-- CreateIndex
+CREATE INDEX "GetGroupKeyRun_tenantId_deletedAt_status_idx" ON "GetGroupKeyRun"("tenantId" ASC, "deletedAt" ASC, "status" ASC);
+
+-- CreateIndex
+CREATE INDEX "GetGroupKeyRun_tenantId_idx" ON "GetGroupKeyRun"("tenantId" ASC);
+
+-- CreateIndex
+CREATE INDEX "GetGroupKeyRun_workerId_idx" ON "GetGroupKeyRun"("workerId" ASC);
+
 -- CreateIndex
 CREATE UNIQUE INDEX "GetGroupKeyRun_workflowRunId_key" ON "GetGroupKeyRun"("workflowRunId" ASC);
 

diff --git a/pkg/repository/prisma/dbsqlc/tickers.sql b/pkg/repository/prisma/dbsqlc/tickers.sql
@@ -76,7 +76,7 @@ WITH getGroupKeyRunsToTimeout AS (
     FROM
         "GetGroupKeyRun" as getGroupKeyRun
     WHERE
-        ("status" = 'RUNNING' OR "status" = 'ASSIGNED')
+        "status" = ANY(ARRAY['RUNNING', 'ASSIGNED']::"StepRunStatus"[])
         AND "timeoutAt" < NOW()
         AND "deletedAt" IS NULL
         AND (

diff --git a/pkg/repository/prisma/dbsqlc/tickers.sql.go b/pkg/repository/prisma/dbsqlc/tickers.sql.go
diff --git a/prisma/migrations/20240815151145_v0_42_2/migration.sql b/prisma/migrations/20240815151145_v0_42_2/migration.sql
@@ -0,0 +1,14 @@
+-- CreateIndex
+CREATE INDEX "GetGroupKeyRun_tenantId_idx" ON "GetGroupKeyRun"("tenantId");
+
+-- CreateIndex
+CREATE INDEX "GetGroupKeyRun_workerId_idx" ON "GetGroupKeyRun"("workerId");
+
+-- CreateIndex
+CREATE INDEX "GetGroupKeyRun_createdAt_idx" ON "GetGroupKeyRun"("createdAt");
+
+-- CreateIndex
+CREATE INDEX "GetGroupKeyRun_tenantId_deletedAt_status_idx" ON "GetGroupKeyRun"("tenantId", "deletedAt", "status");
+
+-- CreateIndex
+CREATE INDEX "GetGroupKeyRun_status_deletedAt_timeoutAt_idx" ON "GetGroupKeyRun"("status", "deletedAt", "timeoutAt");
diff --git a/prisma/schema.prisma b/prisma/schema.prisma
@@ -1021,6 +1021,13 @@ model GetGroupKeyRun {
   cancelledError String?
 
   @@index([deletedAt])
+  @@index([tenantId])
+  @@index([workerId])
+  @@index([createdAt])
+  // index for ListStepRunsToReassign, ListStepRunsToRequeue
+  @@index([tenantId, deletedAt, status])
+  // index for PollGetGroupKeyRuns
+  @@index([status, deletedAt, timeoutAt])
 }
 
 model WorkflowRunTriggeredBy {

diff --git a/sql/migrations/20240815151244_v0.42.2.sql b/sql/migrations/20240815151244_v0.42.2.sql
@@ -0,0 +1,12 @@
+-- atlas:txmode none
+
+-- Create index "GetGroupKeyRun_createdAt_idx" to table: "GetGroupKeyRun"
+CREATE INDEX CONCURRENTLY IF NOT EXISTS "GetGroupKeyRun_createdAt_idx" ON "GetGroupKeyRun" ("createdAt");
+-- Create index "GetGroupKeyRun_status_deletedAt_timeoutAt_idx" to table: "GetGroupKeyRun"
+CREATE INDEX CONCURRENTLY IF NOT EXISTS "GetGroupKeyRun_status_deletedAt_timeoutAt_idx" ON "GetGroupKeyRun" ("status", "deletedAt", "timeoutAt");
+-- Create index "GetGroupKeyRun_tenantId_deletedAt_status_idx" to table: "GetGroupKeyRun"
+CREATE INDEX CONCURRENTLY IF NOT EXISTS "GetGroupKeyRun_tenantId_deletedAt_status_idx" ON "GetGroupKeyRun" ("tenantId", "deletedAt", "status");
+-- Create index "GetGroupKeyRun_tenantId_idx" to table: "GetGroupKeyRun"
+CREATE INDEX CONCURRENTLY IF NOT EXISTS "GetGroupKeyRun_tenantId_idx" ON "GetGroupKeyRun" ("tenantId");
+-- Create index "GetGroupKeyRun_workerId_idx" to table: "GetGroupKeyRun"
+CREATE INDEX CONCURRENTLY IF NOT EXISTS "GetGroupKeyRun_workerId_idx" ON "GetGroupKeyRun" ("workerId");
diff --git a/sql/migrations/atlas.sum b/sql/migrations/atlas.sum
@@ -1,4 +1,4 @@
-h1:wAFKm7rbprHz28Bc8uTDP6nbnjhMbZeAWPtKZpSteaY=
+h1:tROB8K8xtMC6mGl86vNArw3o9Ne5OepxeCRTfi/RH6w=
 20240115180414_init.sql h1:Ef3ZyjAHkmJPdGF/dEWCahbwgcg6uGJKnDxW2JCRi2k=
 20240122014727_v0_6_0.sql h1:o/LdlteAeFgoHJ3e/M4Xnghqt9826IE/Y/h0q95Acuo=
 20240126235456_v0_7_0.sql h1:KiVzt/hXgQ6esbdC6OMJOOWuYEXmy1yeCpmsVAHTFKs=
@@ -45,3 +45,4 @@ h1:wAFKm7rbprHz28Bc8uTDP6nbnjhMbZeAWPtKZpSteaY=
 20240728042317_v0.41.0.sql h1:kgjgRXSRGMCXAOUweAwlTFd/uWtx7a24gLdJEfbK1rM=
 20240809131000_v0.42.0.sql h1:7dk24FIvasOG/vmTI8Oz23C4e6+k3IS9WG5eu2mX/Aw=
 20240812153737_v0.42.1.sql h1:dUADhy9vhbGwdrn+h2KqshcdfSUvSUOlF190Sy6xGhw=
+20240815151244_v0.42.2.sql h1:t1vDmgBTS6QpPIUH+QSgqGIQQ4a7E2g2eA0wHei4jj4=
diff --git a/sql/schema/schema.sql b/sql/schema/schema.sql
@@ -880,12 +880,27 @@ CREATE INDEX "Event_tenantId_createdAt_idx" ON "Event"("tenantId" ASC, "createdA
 -- CreateIndex
 CREATE INDEX "Event_tenantId_idx" ON "Event"("tenantId" ASC);
 
+-- CreateIndex
+CREATE INDEX "GetGroupKeyRun_createdAt_idx" ON "GetGroupKeyRun"("createdAt" ASC);
+
 -- CreateIndex
 CREATE INDEX "GetGroupKeyRun_deletedAt_idx" ON "GetGroupKeyRun"("deletedAt" ASC);
 
 -- CreateIndex
 CREATE UNIQUE INDEX "GetGroupKeyRun_id_key" ON "GetGroupKeyRun"("id" ASC);
 
+-- CreateIndex
+CREATE INDEX "GetGroupKeyRun_status_deletedAt_timeoutAt_idx" ON "GetGroupKeyRun"("status" ASC, "deletedAt" ASC, "timeoutAt" ASC);
+
+-- CreateIndex
+CREATE INDEX "GetGroupKeyRun_tenantId_deletedAt_status_idx" ON "GetGroupKeyRun"("tenantId" ASC, "deletedAt" ASC, "status" ASC);
+
+-- CreateIndex
+CREATE INDEX "GetGroupKeyRun_tenantId_idx" ON "GetGroupKeyRun"("tenantId" ASC);
+
+-- CreateIndex
+CREATE INDEX "GetGroupKeyRun_workerId_idx" ON "GetGroupKeyRun"("workerId" ASC);
+
 -- CreateIndex
 CREATE UNIQUE INDEX "GetGroupKeyRun_workflowRunId_key" ON "GetGroupKeyRun"("workflowRunId" ASC);