diff --git a/api-contracts/openapi/components/schemas/_index.yaml b/api-contracts/openapi/components/schemas/_index.yaml index 2c13dfc4c..5e5d178e7 100644 --- a/api-contracts/openapi/components/schemas/_index.yaml +++ b/api-contracts/openapi/components/schemas/_index.yaml @@ -114,6 +114,8 @@ CancelEventRequest: $ref: "./event.yaml#/CancelEventRequest" Workflow: $ref: "./workflow.yaml#/Workflow" +WorkflowUpdateRequest: + $ref: "./workflow.yaml#/WorkflowUpdateRequest" WorkflowConcurrency: $ref: "./workflow.yaml#/WorkflowConcurrency" WorkflowVersionMeta: diff --git a/api-contracts/openapi/components/schemas/workflow.yaml b/api-contracts/openapi/components/schemas/workflow.yaml index 8d28e3035..5b3a6005c 100644 --- a/api-contracts/openapi/components/schemas/workflow.yaml +++ b/api-contracts/openapi/components/schemas/workflow.yaml @@ -8,6 +8,9 @@ Workflow: description: type: string description: The description of the workflow. + isPaused: + type: boolean + description: Whether the workflow is paused. versions: type: array items: @@ -27,6 +30,13 @@ Workflow: - name type: object +WorkflowUpdateRequest: + type: object + properties: + isPaused: + type: boolean + description: Whether the workflow is paused. + WorkflowTag: type: object properties: diff --git a/api-contracts/openapi/paths/workflow/workflow.yaml b/api-contracts/openapi/paths/workflow/workflow.yaml index 00527927b..1d8ccc70c 100644 --- a/api-contracts/openapi/paths/workflow/workflow.yaml +++ b/api-contracts/openapi/paths/workflow/workflow.yaml @@ -116,6 +116,49 @@ withWorkflow: summary: Delete workflow tags: - Workflow + patch: + x-resources: ["tenant", "workflow"] + description: Update a workflow for a tenant + operationId: workflow:update + parameters: + - description: The workflow id + in: path + name: workflow + required: true + schema: + type: string + format: uuid + minLength: 36 + maxLength: 36 + requestBody: + content: + application/json: + schema: + $ref: "../../components/schemas/_index.yaml#/WorkflowUpdateRequest" + description: The input to update the workflow + required: true + responses: + "200": + content: + application/json: + schema: + $ref: "../../components/schemas/_index.yaml#/Workflow" + description: Successfully updated the workflow + "400": + content: + application/json: + schema: + $ref: "../../components/schemas/_index.yaml#/APIErrors" + description: A malformed or bad request + "403": + content: + application/json: + schema: + $ref: "../../components/schemas/_index.yaml#/APIErrors" + description: Forbidden + summary: Update workflow + tags: + - Workflow workflowVersion: get: x-resources: ["tenant", "workflow"] diff --git a/api/v1/server/handlers/workflows/update.go b/api/v1/server/handlers/workflows/update.go new file mode 100644 index 000000000..af78f43b6 --- /dev/null +++ b/api/v1/server/handlers/workflows/update.go @@ -0,0 +1,31 @@ +package workflows + +import ( + "github.com/labstack/echo/v4" + + "github.com/hatchet-dev/hatchet/api/v1/server/oas/gen" + "github.com/hatchet-dev/hatchet/api/v1/server/oas/transformers" + "github.com/hatchet-dev/hatchet/pkg/repository" + "github.com/hatchet-dev/hatchet/pkg/repository/prisma/db" + "github.com/hatchet-dev/hatchet/pkg/repository/prisma/dbsqlc" + "github.com/hatchet-dev/hatchet/pkg/repository/prisma/sqlchelpers" +) + +func (t *WorkflowService) WorkflowUpdate(ctx echo.Context, request gen.WorkflowUpdateRequestObject) (gen.WorkflowUpdateResponseObject, error) { + tenant := ctx.Get("tenant").(*db.TenantModel) + workflow := ctx.Get("workflow").(*dbsqlc.GetWorkflowByIdRow) + + opts := repository.UpdateWorkflowOpts{ + IsPaused: request.Body.IsPaused, + } + + updated, err := t.config.APIRepository.Workflow().UpdateWorkflow(ctx.Request().Context(), tenant.ID, sqlchelpers.UUIDToStr(workflow.Workflow.ID), &opts) + + if err != nil { + return nil, err + } + + resp := transformers.ToWorkflowFromSQLC(updated) + + return gen.WorkflowUpdate200JSONResponse(*resp), nil +} diff --git a/api/v1/server/oas/gen/openapi.gen.go b/api/v1/server/oas/gen/openapi.gen.go index 583bb1a86..e969c8741 100644 --- a/api/v1/server/oas/gen/openapi.gen.go +++ b/api/v1/server/oas/gen/openapi.gen.go @@ -1121,6 +1121,9 @@ type Workflow struct { // Description The description of the workflow. Description *string `json:"description,omitempty"` + // IsPaused Whether the workflow is paused. + IsPaused *bool `json:"isPaused,omitempty"` + // Jobs The jobs of the workflow. Jobs *[]Job `json:"jobs,omitempty"` Metadata APIResourceMeta `json:"metadata"` @@ -1290,6 +1293,12 @@ type WorkflowTriggers struct { WorkflowVersionId *string `json:"workflow_version_id,omitempty"` } +// WorkflowUpdateRequest defines model for WorkflowUpdateRequest. +type WorkflowUpdateRequest struct { + // IsPaused Whether the workflow is paused. + IsPaused *bool `json:"isPaused,omitempty"` +} + // WorkflowVersion defines model for WorkflowVersion. type WorkflowVersion struct { Concurrency *WorkflowConcurrency `json:"concurrency,omitempty"` @@ -1593,6 +1602,9 @@ type UserCreateJSONRequestBody = UserRegisterRequest // WorkerUpdateJSONRequestBody defines body for WorkerUpdate for application/json ContentType. type WorkerUpdateJSONRequestBody = UpdateWorkerRequest +// WorkflowUpdateJSONRequestBody defines body for WorkflowUpdate for application/json ContentType. +type WorkflowUpdateJSONRequestBody = WorkflowUpdateRequest + // WorkflowRunCreateJSONRequestBody defines body for WorkflowRunCreate for application/json ContentType. type WorkflowRunCreateJSONRequestBody = TriggerWorkflowRunRequest @@ -1835,6 +1847,9 @@ type ServerInterface interface { // Get workflow // (GET /api/v1/workflows/{workflow}) WorkflowGet(ctx echo.Context, workflow openapi_types.UUID) error + // Update workflow + // (PATCH /api/v1/workflows/{workflow}) + WorkflowUpdate(ctx echo.Context, workflow openapi_types.UUID) error // Get workflow metrics // (GET /api/v1/workflows/{workflow}/metrics) WorkflowGetMetrics(ctx echo.Context, workflow openapi_types.UUID, params WorkflowGetMetricsParams) error @@ -3705,6 +3720,26 @@ func (w *ServerInterfaceWrapper) WorkflowGet(ctx echo.Context) error { return err } +// WorkflowUpdate converts echo context to params. +func (w *ServerInterfaceWrapper) WorkflowUpdate(ctx echo.Context) error { + var err error + // ------------- Path parameter "workflow" ------------- + var workflow openapi_types.UUID + + err = runtime.BindStyledParameterWithLocation("simple", false, "workflow", runtime.ParamLocationPath, ctx.Param("workflow"), &workflow) + if err != nil { + return echo.NewHTTPError(http.StatusBadRequest, fmt.Sprintf("Invalid format for parameter workflow: %s", err)) + } + + ctx.Set(BearerAuthScopes, []string{}) + + ctx.Set(CookieAuthScopes, []string{}) + + // Invoke the callback with all the unmarshaled arguments + err = w.Handler.WorkflowUpdate(ctx, workflow) + return err +} + // WorkflowGetMetrics converts echo context to params. func (w *ServerInterfaceWrapper) WorkflowGetMetrics(ctx echo.Context) error { var err error @@ -3906,6 +3941,7 @@ func RegisterHandlersWithBaseURL(router EchoRouter, si ServerInterface, baseURL router.PATCH(baseURL+"/api/v1/workers/:worker", wrapper.WorkerUpdate) router.DELETE(baseURL+"/api/v1/workflows/:workflow", wrapper.WorkflowDelete) router.GET(baseURL+"/api/v1/workflows/:workflow", wrapper.WorkflowGet) + router.PATCH(baseURL+"/api/v1/workflows/:workflow", wrapper.WorkflowUpdate) router.GET(baseURL+"/api/v1/workflows/:workflow/metrics", wrapper.WorkflowGetMetrics) router.POST(baseURL+"/api/v1/workflows/:workflow/trigger", wrapper.WorkflowRunCreate) router.GET(baseURL+"/api/v1/workflows/:workflow/versions", wrapper.WorkflowVersionGet) @@ -6821,6 +6857,42 @@ func (response WorkflowGet404JSONResponse) VisitWorkflowGetResponse(w http.Respo return json.NewEncoder(w).Encode(response) } +type WorkflowUpdateRequestObject struct { + Workflow openapi_types.UUID `json:"workflow"` + Body *WorkflowUpdateJSONRequestBody +} + +type WorkflowUpdateResponseObject interface { + VisitWorkflowUpdateResponse(w http.ResponseWriter) error +} + +type WorkflowUpdate200JSONResponse Workflow + +func (response WorkflowUpdate200JSONResponse) VisitWorkflowUpdateResponse(w http.ResponseWriter) error { + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(200) + + return json.NewEncoder(w).Encode(response) +} + +type WorkflowUpdate400JSONResponse APIErrors + +func (response WorkflowUpdate400JSONResponse) VisitWorkflowUpdateResponse(w http.ResponseWriter) error { + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(400) + + return json.NewEncoder(w).Encode(response) +} + +type WorkflowUpdate403JSONResponse APIErrors + +func (response WorkflowUpdate403JSONResponse) VisitWorkflowUpdateResponse(w http.ResponseWriter) error { + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(403) + + return json.NewEncoder(w).Encode(response) +} + type WorkflowGetMetricsRequestObject struct { Workflow openapi_types.UUID `json:"workflow"` Params WorkflowGetMetricsParams @@ -7125,6 +7197,8 @@ type StrictServerInterface interface { WorkflowGet(ctx echo.Context, request WorkflowGetRequestObject) (WorkflowGetResponseObject, error) + WorkflowUpdate(ctx echo.Context, request WorkflowUpdateRequestObject) (WorkflowUpdateResponseObject, error) + WorkflowGetMetrics(ctx echo.Context, request WorkflowGetMetricsRequestObject) (WorkflowGetMetricsResponseObject, error) WorkflowRunCreate(ctx echo.Context, request WorkflowRunCreateRequestObject) (WorkflowRunCreateResponseObject, error) @@ -9232,6 +9306,37 @@ func (sh *strictHandler) WorkflowGet(ctx echo.Context, workflow openapi_types.UU return nil } +// WorkflowUpdate operation middleware +func (sh *strictHandler) WorkflowUpdate(ctx echo.Context, workflow openapi_types.UUID) error { + var request WorkflowUpdateRequestObject + + request.Workflow = workflow + + var body WorkflowUpdateJSONRequestBody + if err := ctx.Bind(&body); err != nil { + return err + } + request.Body = &body + + handler := func(ctx echo.Context, request interface{}) (interface{}, error) { + return sh.ssi.WorkflowUpdate(ctx, request.(WorkflowUpdateRequestObject)) + } + for _, middleware := range sh.middlewares { + handler = middleware(handler, "WorkflowUpdate") + } + + response, err := handler(ctx, request) + + if err != nil { + return err + } else if validResponse, ok := response.(WorkflowUpdateResponseObject); ok { + return validResponse.VisitWorkflowUpdateResponse(ctx.Response()) + } else if response != nil { + return fmt.Errorf("Unexpected response type: %T", response) + } + return nil +} + // WorkflowGetMetrics operation middleware func (sh *strictHandler) WorkflowGetMetrics(ctx echo.Context, workflow openapi_types.UUID, params WorkflowGetMetricsParams) error { var request WorkflowGetMetricsRequestObject @@ -9319,184 +9424,185 @@ func (sh *strictHandler) WorkflowVersionGet(ctx echo.Context, workflow openapi_t // Base64 encoded, gzipped, json marshaled Swagger object var swaggerSpec = []string{ - "H4sIAAAAAAAC/+x9+2/bOtLovyLoXuDuAs6z7fnOBvh+cBO39TZNsnZyiv0OioCWaJsbWdIRqTy+Iv/7", + "H4sIAAAAAAAC/+x9+2/bOtLovyLoXuDuAs6z7fnOBvh+cBO39TZNsnZyiv0WRUBLtM0TWdIRqTy+Iv/7", "BV8SJZES5VecRsBiT2rxMRzODIfz4k/XixZxFMKQYPfkp4u9OVwA9mf/ajhIkiihf8dJFMOEIMi+eJEP", "6X99iL0ExQRFoXviAsdLMYkWzhdAvDkkDqS9Hda458JHsIgD6J4cvT887LnTKFkA4p64KQrJb+/dnkue", "YuieuCgkcAYT97lXHL46m/JvZxolDpkjzOdUp3P7ecN7KGBaQIzBDOazYpKgcMYmjTx8G6DwTjcl/d0h", "kUPm0PEjL13AkAANAD0HTR1EHPiIMMEFcGaIzNPJvhctDuYcT3s+vJd/6yCaIhj4VWgoDOyTQ+aAKJM7", - "CDsA48hDgEDfeUBkzuABcRwgD0yCwna4IVhoEPHccxP4V4oS6Lsnfxam/pE1jib/gR6hMEpawVVigdnv", + "CDsA48hDgEDfeUBkzuABcRwgD0yCwna4IVhoEPHccxP4V4oS6Lsn/ylM/SNrHE3+hB6hMEpawVVigdnv", "iMAF++P/JnDqnrj/5yCnvQNBeAcZ1T1n04AkAU8VkMS4Bmi+QQKqsIAgiB5O5yCcwSuA8UOUaBD7MIdk", "DhMnSpwwIk6KYYIdD4SOxzrSzUeJE8v+Ci5JksIMnEkUBRCEFB4+bQIBgdcwBCFpMynr5oTwwSGsL7ae", "cRjeI8IXbjkZYj2ciH3lPzNqR9hBISYg9KD17GM0C9O4xeQYzUInjXNWajVlSuYWpEXJok+bPvfcOMJk", "Hs0se12J1rTjUxCF/TgeGrjyin6n7OYMz9hqUgxZH8r1lIqIg9M4jhJSYMSj43fvP/z2X7/v0T9K/0d/", - "/8fh0bGWUU303xc4KfIAW5eOKijoAi7oO3RQ7ERTh2IWhgR5TNCpEP/pTgBGnttzZ1E0CyDlxYzHK2Ks", - "wswmsIf0BEiAFPslaRJSAVbDtYJysiGoNBSdnChkkluhqyohMXGoxQ39QhHCh8hhrEr3RnEqZK5cTI0M", - "u8qJtCTKYvQlwsRAgREmX6KZ078aOnPaSoVxTkiMTw4OBP3viy+UOHXHD4jRV/jUPM8dfCpME8/vbnPS", - "BRPPh1Nr8h1BHKWJB/VinMtEv29YPUELqByKiRjLeQBYiNOC1HaPD4+P946O947eXR8fnhz+dvL+9/3f", - "f//9f1xFTfEBgXt0YB2KkEEQIJ/TiwJEz0Ghc3PDBQMdWgVkMjk+ev/74X/tHb//De69fwc+7IHjD/7e", - "+6P/+u3IP/Km03/Q+Rfg8RyGM8rc737TgJPG/rLoCQAmjui/ThyV6B/RwfNdVEE28MJ1dAd14uAxRgnE", - "uqV+n0PO7pQ4Ce3uiNb71hu7gAT4gJNgwxlRoFijHLkuyZEMtv3ivh5/+NCEwwy2XiZOMmRokeh5MCZc", - "JxjBv1LIhUcRn1wB4JhdjSoXKDQTac993ItAjPbo5WAGwz34SBKwR8CMQXEPAkT3xT3JVtxLU+S7zxVC", - "4vDq1vsxDe64zjW4hyExLhney7uPlX6qGbJRU+Uz/NABheMoxLAOqioB8W+UYqwgZjNVgVyBus2UqCz1", - "lB6xgQXuh34R+60pL79LpkywtKFEq72jELIlsZ2T0si8Ks6Hw1C/fX6a5HfGhzny5kwUcBGFsMOof99d", - "nmeiBSIhCnpyIrYovTzqc2nEVe6VxBEbX8eHZaSZKJ5ICV/FWAGsejD4KGY46ukR+D6i84Lgm8IbJZRl", - "bRxJ+Bn+GLUoQOaz68di5GA3wJ1OB6P97+CTsbsBSVxVYyDlZD2+GCuatxFFJIqR109MO7UA/xuFjhQX", - "zgWlrr/1Rxd/lyfe+GLssDFWofDsVFig8L+Pegvw+N/HH36rHg8ZsGaC4BfyfgATMlgAFHxOojQ2szZt", - "gnV8FCBM6Bp5C3ntS7BrfSdaYvk+uoc9NmN17QLUppU3KAR8cP0xRD/JbaVrdUgkLAhr2Vu5rp6bRAFs", - "OqT4ar7BxQQmI9peiw9XDNaEFSM+7NQ6bqlZBxbYMnCQzvST0i/rn7QnrJGUe6uEJbRNBpQOj1zf2KJs", - "XUk0rqTkk8ya10yaeXud3YiCOzwrbmXZiitsvMaFPETJ3TSIHkZpOE4XC5A8WamG36vdajQ8fnZkC/kh", - "N/wM6G7qbY4952//HF9eOJMnAvHfmw+x7Phi039djQbkGOdIx/QxmKEwM0jVIfQqa5kpOkx+PdhfMLLl", - "VFVTCeiuQFkD4mXiw+Tj0xlKoCdBgmG6oDsHsOdy744iP0p7Ifp/kr4P2Tc3Ghi7jiFIvLn2vDXRewWX", - "U4C0VkYm6FN6xlBW5a2cJA2LNgWzSyuGoU9haRhYNGsz8l8pTJsh5q3ajJukYWgBsWjWZmSceh6EfjPQ", - "WUP70Skd/jOaaARSnU+RySXFqyik8X+iyf6GrEOVMTGBsT0XjgmMdTd79ZypKp1oAaOU6JcvPjYt/R4m", - "GEWhdgbz2ZGBpQ6Qma/40nUaxT+jySjVWP88ZmIIpKnTzqaXdcqc2+YmIwgwJxSNVzZEeN5u6v9wiqzb", - "UUq0vKVh91YgugTiNCDKqDmGMQEJabcYTABJscV6qJzlbQV9j9KwHYnTzW9P5d4dTOpZoM1yFeWqCWTl", - "gCn1XJ5fioNIAsl2wcw142yb5BF6Nbg4G158dnvu6Obigv81vjk9HQzOBmduz/3UH56zP077F6eDc/q3", - "7qylSojeY2fr5y931WyxmISZjrDZdrRV1SfzRmi1Hwpx0ZSCXxjeIjSNFk8FNjGRjrjYMgPg3X2Hk3kU", - "3b34IhVY1rXEaHaOQtjK/UiPUPaZqg9UnsiDNIhmToBC2Mb3xGOUtHPQ4USDRtXE1Ju30NyoS9hS/XR5", - "4FQ2w48cVefwHgaqqDkbfLyh4mV48enS7bnf+6MLt+cORqPLkV6mKONkqr/V/hcg0AkS8f3lb06SrPTS", - "g39c4fZUHKHl/Ul0rrlBaRCgumx+ul6aJDAktzGj3eOeG8JH+a93PTdMF+wf2D05OqQXoiJnFTrrnNei", - "hRNzKswmPra6ciiwaCM84GN15Hd2I+fr0vrcIwIC9YJHmzK7RIAw4Va6PELy0OaGo5FY/6K3u2+QJMjT", - "yOMwXVzZXT8ZHctL6L5pvf+yunHysRB3xbPrp3HAkd1Vk48oLpz7etQU7JUZqIVZeipCdPJ/BAg8Rwuk", - "ERhWFseEiv+ADqAV0QHAZASnKDCY11kohojVUAdjcRoJ6wiZm3CNgSxsgj9AkBqOnQV4RIt0od77uaEc", - "OyzmTxgqxW4/oNCPHvTbvQ5LaAOC783rkFJEs44F8KHtIvg3/RT8G1sG3UMUKq7eHM08Om0aJR70bf12", - "yq1A2S+53gyqAoX9UOl5Bw7BnLe0x2D2eYWDsDxG5Sjk2JRYU1CpHQ16MCRj5fZa8m4w8Ez0zL86Ore+", - "am5ocx9dxv6wgu1gYwYCgdLcQlC5LpfDO+p5JNuInnqTFrCUR9eKfUj/ejtxUiMYB+Dpl4rT4UtSzDDY", - "uLICPbzs+pTmHw4PG9Zbgtu0apPBROluL7RLdi1b+CR0CeVyxuw1bBWnOgNlhZppMzpqybahGXAGMblJ", - "DDrWzejcIZGDYeizwBRxvcUOiTbjKjYdEGmI/qLagA9DgqYIJpkWKRQgEa/L42fU8PYJDKJwJiFukJW9", - "TYbv2Bkya0NyxnAB4nmUwHEQkTWfsoUTTO/44GolDiKuZIse9iabJU88YRM3LYt+ptcdsbDmLVaN280L", - "RUEgvT72K60c0hoFWDSxB71EPzlaeuqpXraESws4JR/VCFg1281BGMLABK/47CBff9vAdHDngY+u1+P4", - "CBfGECE5BQsVWnKSlUQQWJhWT7+tsHTa3bxuNvgqi94J4Wkn3iQiMnQX6aKnkKFWAhIYm+Se3lc5R4Gf", - "wKLjpUF32pB/MQZJJZS+EZIEAh9MAmjaXPk9S6bhArGRTFZyextmMFOAsooCOUg3ndhAboGs2foNuLn7", - "ZBBHBWuuYsFYkzOcEeF3k07ZSAOF7vg0SkOiBxcaoVzmOpz3qcFQWSUtePMtnMEidiFrv362i1JiAnFJ", - "jmRm2v6UwMQemWsPLuBdanZmBW3LNq6GtjWJEwtZ02bFWZeaFVPVxxDTYHU4ZRSYraw2gECgrp94c3QP", - "X6VcUj21dvDtlIiJEh8m+k41XJ9AkjzVSNGN8aNyjdkOS9TcGBQkSDz2tN5PE73vgJW8xIBaU7loYwi6", - "98xUIE+bChqQr++gBCRoSE7yoMV6hK2R9aB0A+9hgshTm95j2ceK7j6hBJMx5EqyPe2dg7a9WoZ68VtG", - "AcDSzBlmFTSpURh8f2uIeVfixQtk2kjIuUiXzprR4F83g5vB2e3F5e33y9HXwcjt5T+O+teD2/Pht+G1", - "23PHp18GZzfnw4vPt9fDb4Oz28sb+nN/PB5+vmCBbePr/uiax7oNL4bjL8Wwt9HgevRvHhaXR8D1XDrW", - "5c317WjwaTQQfUYDZVR1svH5JW15PuiPszGHg7Pbj/++vRkz2OkiPp1ffr8d3Vzcfh5d3lzdfh38+1YN", - "xDM0EYDqvFRaFlGwqMThiAWOhtfD0/553Wh1EYTir1uOhm+DixKmrSMMs79pax0weeGcckkfmIi0r4Eh", - "Oe+7LA0SOay1NAssWC+8r60DAkIQPBHk4cuYXKakZtTczjAH2IliAn1H3CWzQfRzbLy8gCklbOWcsuZi", - "BMb0MG3C5XYzLTeU+25OuNSueQeksn4vdImps2iPk5w7ohMwia30RuFsDAn9D94ei/LyOYPHGNFdZjG5", - "DJj68XkvPg12HliVEBZe7IAEOiCOkwh4cxTOeLkQhuC6+WXCKCcSFnGwJBR8ybIeSxUeFqJQiwvFBPMJ", - "oCBNoAUozPulAqJa7jFLctLPGQDMl2r2quRBTCAUO8s8KyIPzzJsATxKIvvEjBOh92SMT3KmsokDiIy5", - "EVS1XoO6WRJoATbLhWEWTLCZ3OvnrDRMrUdIFgQSReC2WSxnuQTvJr+AYCiTV0N+NmONt6jza7ARCiU0", - "ljgxC5np+V6pWbYNtLMzR4kg5XYnCN/TKvwvRlD2Cd2U9Zpa32CY8B5X6SRAXh0psPFqahSoMO/Mpov9", - "W2bTR2Kf5M3i8vsFux31z74NL9ye+23w7eNgVHMhqA95ZobsUvWBq2I5k2pkahnnLHa7CRMFOBRLQN3c", - "bcYrB8RkCJCUr2IxuyAP/uA3MvUmyW59lxf5RXlQg96CWqPT7ECyqIkXZt8dFmKpl8E8oplEzgNIWL5t", - "Rd/hvfXxt+1CqPXR0+sJjOZjm5eoh3+1XM5s25s5NCMSu7Dopg1rHw29gAQmMiZaHpV8LOdvaB/uO0eO", - "D556zpHzAOEd/e8iCsn870u64TP0aGOkzZJVIuoqCpCnqQvAVfC6W2lWTpE31egFLSRrkf2aYu4EcNrV", - "JWg2g4lyP2hZA6tqJW4bqnfDSjy+xXJL6sob4orXUunIqD2ogJj3/xUb1DqLwMtaBDZ4U99I+UNre+mz", - "kZu+M5+8OaIZX4EU69LkVHLnjn0HYSdmrR0Q+o4HwjAiDmB1W1kBeFn+pIx4LXRYd6VqNCkA308gxqpp", - "oaAlybtq1cJAP3wBeK6T1nOA5+qQ/w+XphPymysavH76mJcid07ngBgn/AMmaIqa0MsMJFSW3IvmooZ/", - "AQY9Rc8BNr8UoJ0DZE8DOBiSLRr+fYTjADwVCFruX2tbRBG7PwwEVnxKwVw0Dz6Ykch4ED7kWJMakx72", - "JY7t7KmGZxYSVQdIBkQt/laDoVKMIHtIQsWTCeXn0QyFy5drXI6/V6reuHMYl2uMm3A9gjOESY1030V0", - "2510BsGwg7sli5vbbpqqHuM5ivFrtZNV7IZbPM03ccrwyXTbJlI2uCq1VjuwHTOI1AOhhmnZIjWlkMm+", - "aRIs4yan4zaihFemXbEmrcUiMfQSaPDk8W9ZgQPBw/Qm5Ayn7CWfOInukQ/9ngOcBIR+tJCdWI7RBDoz", - "GMIEiJQqNR3yeGMYb49mfzcJcLm92TYpZ3A2IptK5R0p6FUUP1ZPFRW6GBlThJveAmKsgQvZVS8v88GH", - "Up+qaeWAnUd+q9UK0L/xnllQ/an2gT8K8pfr6yuHN2Iv+0kKTgTyLeqxKFjJYC5M/MMS4fUkJCt6mAz2", - "3H4oaV62tjbQailgadr5lm2d9OF8Hly7Pffqcsz+c3PNbKimE5KnF+G6tFjM7ffC0uCB0IlhQulqv1Xc", - "FLgHKACTAMosn4aisNVp4SP0UgIdLwqFvyF40jsUqKrBCpEnw4YHlpizAs1C6Dt5p3U8tbRiSn8AJjDA", - "9c4W1oaxVH4cZMeAdV4+TM7pOLotCwAmXyBIyAQCi6xgsVXMd4YpgMCZy97rLjcEOBNTtWCACZgELFli", - "hyBcgEczoWuqIa1G8JvXM8z6RVIpcKPLxaRtsoT03LnVkmBLxXQ0NItl/r9O2RE1Anj+OmecJQEp1RvQ", - "AZIFP+sS89kxWMGtFOH90+vhHwNW/jD786p/MzYEb/Mf8hNgPDj/9OVyzAPVv/Uv+p95VPjg45fLy6/a", - "IcRpZkzJF4edeJayCHVjXQHR+6ZJfbwZnWuGb6tNsvZaTUCRdu0Ks8lSnLTrunPra7zq3JveMHn9gws1", - "eHh544ZRbxZAyjCY1QuxSzfUvqG4toFP6RfdEFarEwW51xY/3kZUG9cqLWmaUxLMll+r3K1roNX9RM32", - "9gWD/uAdszoUdUEMZUlgIn067qk8UXWhGjNIlO9ZlkHJNRXKqjHc/ziDBDPceXlXZ0b7ZiJN8ajuG0OF", - "xiQBBM6eTAcH/+qQiHu95FOV6qw8pIg90Am8uSj1KI4GnitzO7y4vRpdfh4NxmO3556NLq9uLwbfB+zK", - "wDKj8n/y9KHR5c3F2e3o8uPwQnt+tFR3co2m6CEuP9Hw7rj5ViinLiOwp93IOqoYnulckBmAwzPttsne", - "X1FYuId9urk4vR6yeLmzm1H/4zk9xc/6+iQldRApmVtxCptdw3ryu17cr1SzY8snBTsN7O7JorUxupPx", - "5VeYpzprxGGp7m6Vre/gE9Zr43J4SpY1U5S0fyomgINj6KEp8vJJnL/FAGPoO/cIOFMUEJj83bKs7/fi", - "0wM24Vr6kNPC4+nK24TcpH9hepMkC6RQi+IdHSo16jZWoiOLMmu1IF7nwJ4u8xodazzmee0Nropv2+7A", - "5x6ridHbBmFjBUPV55uyoh711Th48CP0Pz61GPxa6aUEdAtdpqXqoxlh9bdA/lDe0MkKnKqL/VEvTHbk", - "7lBXUrIO/LrawP3xKT2mB+PT2nM6H6WmUL5KywUppkjGhknGcxDDTnZ3sruT3S8puw1z/IKifS2vPolL", - "5FmjdGOTLXXfKRKC4dJT2lCNmzUKrxSO1ZSBisIxvUOngV5KiurRmymI+H3J978athjzB/aXqWy9yULc", - "5cLUDYswXu5YZZs2dCSHOuUdm7SHUvPK/IIftCmCkpe0HwXPaL9J1tN+zLlRX+nKuJprMNPhL+Bn+eqW", - "1ZUNlvqYHA5hHYEIrj9NqIY51TN+Td3DW2Rgt6YJRQ2iqaEQ/q3wMqx7WqxfYXttuoQ3jWhl61h64Aw/", - "69W6+DmoR19+NN4KM3R7NCvne5lXCnZkG0yopmeqZsMpSANylaBI1lzS8R1r5MSilY5zGs2muaPjhdwX", - "WU1CC1CxOHSv89q7Gs0ReXdGQzn9ltvLrXwjCjO1oGmseDgMfjz+0QoINf3b1iJaq6WatUcJc17lUBno", - "RzM7sH1dp0m5DYG8KYRzR2luSy49OZ5AFoZQU7hzAR4bWrQsQGgqH8jjTVMqpKjevOAQTiBIYNJPCcvx", - "Yhhld1L2c74pc0JYDSkviu4QlM0R3VX+k3TtnbhzFqmlpHeBGLGn75+Z/WAaGYIQeTenfzVkBRMJuxsX", - "f80oyz3aP9w/ZIQZwxDEyD1x3+0f7R+yZAIyZ0s7ADE6CESV25kuzvaz9AzSViHE2MnuZXQXgXyYwj0X", - "3z+zdcm4SDbL8eFhdeAvEARkzqTyB933i4hkcxZ2xj3580fPxfIJewph3lD6iP8U43tz6N25P2h/ttYE", - "Av+pebG0Gapb7Ug2WOdyGXAsF5TnPpIETKeicErd6jNoG5d/f3QARKLqHstL2GOOGnzwk/2s/vbMYQwg", - "0SjBZ+x37IDsoTqWD82zL1j3CsZKue98BEaLCWBlEijYNdWGKjM47A7H+IvSc85dlaW4Kvdz+xuXiytf", - "Cp9/VPb+fRVb49TzIMbTNAieHI5Sv/DKXwV5zz33PacSLwqJKHEL4jhAHsPowX9EndB8HQ2nFSsoLTJs", - "yj7iBQgoFqDvRIkzAb6MCuZgvFs7GDooPkXJBPk+5Gm9OX1zOqkjM0nxojrRj577uJeljtMPorhRT0MY", - "P9jthXia7F2e5rwKifMRfg0SZ/TwMeKycy3EYFEXQ0MmtdgikZNKnBex8awX0WtZiKGYZBX2ghjggHZi", - "wFIMcGrZnBhQD8gY7fE6GAc/s7/ZaRhHWKM0jOB9dMcKPfavhryChoiGyGYsiYkYsRIdfE28u42UyIY3", - "yAQJ604ddwlbnqBzBt2vTdS4DVUL0qEbey12TpJx/lsdJWdbXqBgL4hS/0C9ypq1Xdkqi/OT1wk2iINC", - "TEDICkoVifiUfpbuW7MSvHncMkCcNMwydHaGwBq0do5g1R8mtv6b4gl53JND7EUxdyaLE03Zb27VPPjJ", - "/vtct99USrFW+5UNZcZNvpGNkogNYVRO2NetCqH1bbYo1N9weCeQJAjeC7HGscF2rJNtBRJXMJOTN0dx", - "jVTj9PPDTOEHTWKNbUsm1Rpo/iwTYG+d7s8YCXe0v1u0v4BLn+HG03t7B7eoGd6GprIj8ZUc5Os4wukY", - "B8oTndi44+cI0wtQ4BRamzaYth4WG25st+lcYsfVx6nbbb5MUC+sbpcIIdt6thGlTajuv7rJ7MHXg5/s", - "PxbmVWesPhBb2WL11V97a2phTONRxkDcSbNpESe7dOYcbQeMmxCkZB4l6H+hzyf+sJ2Jea0GVvIGBEH0", - "AH29qbZMtZIn2O91Zx8nuiLHhPjgJw6xFbcUHzmu8kuIW7BJ6cVkI6MIkbpzbFJCRscoO8goFYLNWOVi", - "XMsoIdawCf/8rJoB9AZLOq+8q1RYpLXTwsQZGbSbYo6e+YZ2B5+WvaIpMBx/+FAA4sj6SlbDoHES0X9A", - "vzvDdog1Tdo9IvN04oA4ltRePdZ4mxI/EhjvJSk7vMSfzweAP2fapNmLVjKhUtQcqbIqT5RgOrcc2IJp", - "5XjmA03Au23GFemkJHLwHYolbH+lMHnKgYumU8xurBpQUEh+e6/NLK2fjuebT54MU7LPLWfcpKFG81jv", - "EhYb/MatNXTW99uZtcB1DwAz4TON0tDX3ScL7K8wf6YZ0J9Gaa1XKGPhZpmUx0ObJRJv00IeDfignTR6", - "M9Iof225k0W/jixSGH/zkiiIZvVyCDtBNHMCFFZ0o6pf5zyanaOQn46dGNoNMdQzP2YUwHsYYDovLxBS", - "MzFrWZi51iIt6ID24pnuhpVjSA9eh82mwDGNEgMgvENbQMa8lwaI7+wZ0shhofXm9Udq1n7LyQsZ/wY8", - "8On9rLRALRRnSrNlIMn7b/aQUqVB0/lESbI7nAxuTXYqZFJYOQvOo1n7Y4B/xmY7Fa9Ljh3A3kkxBNPx", - "cD/e1N1MpCofvPh8WH1oKolEFesXCERtJHFR4UOJPO3iTDMS53udE1tTVKmOojNTLC8rUxNdzkJTHhEm", - "KJzVE/jrMctuIVzcjgnzNLMXDQzv+HFtcd8torxr+VKfA1UfYwMybdUUg46b8kFsryM7wcHbTJZYwnJg", - "3oSOdwrqWh212jNTr4WK1j5RKtPe3urhpmqY68uFslZBj144F6p6Ana5ULY66kq5UHan5AGGhP4XN+dN", - "yy6O7FKfCaWQCwpnY9HHMhj7jRyTCmJWOCPVPelYqRC+a0TT2vgoSyisd7Rl+X3YLn+w0yezmGOGD5yX", - "Y23FJ7J4SGfrKyuPWRIibpeZ2KQwLpEs2+mIDAGS1hW1cJMmjPKkHX+ti78EIyyZ+lt/4FhEdWCWQlII", - "7eC9DUlyr+Wsectu1Dv4ZOVEpe0Ks1pV1GNkwIpUVSuhmmFSnh+xgi2XFa0BVN5BWQ7EJA1FuSdoBats", - "a+3+1NcOfiGXNNvPl3FIs6l3wB2twqE6o2uIJUu1vINP4oWzGKCkQi9Z6fI/KbsdnbCmR/xZs2P+r2Mq", - "3nXr0ZTH1zJDY4Fi8zJkIrMVnYsq0QaWXG9R5Y3nOHdRAGu5GUAZ42mZ2WxrQq5L1O+uAAwBogpxrVmY", - "8/fLhCHYldBQbb48K+fNR4Ee/2M7s8rCtUI9hY8ehH4lSU1cUGTGlDWfN19MDiZpcGcO+/mYBneCPHAu", - "E3CtUKB93rBgoMtvKRzwC0mHCqiWJoWKvOjCxndMYDC+VaUGXrPY8NjDHzXxguw7t2wobxIWdF6TGOFx", - "JnyEt6xhMATYaxjiBpHAOABPa5cjL/awS7kseINoYkiDfvFt+E5I7aKQGjFK3Yx8YnY1S6MrN9ZZGF6/", - "wqfOz5dbH5e6vjNkd1d43RXeEcbgdfKBOA1qCubS77jd0TySR8xbPZo5AnblaF6PnY0D12n1b+3AROE9", - "IrBtxLXspY8iG7Kv3Vkpg8cUfCwVNiax3QWL6eKpc1rcUBA1n6CW1jt7uBI2zVFiFy3NcfuiIdIc3GUi", - "owVhdGypD4fO+GY9sZuCz+UPe/zf7d5GsmDl1q8h7VaATZGv6mHby9Dx2s/WRu7VPPW0Y9yrK0uY7Y8p", - "nbu4j22eULLghFdef3AHOWGzubjLnbsvlo1rybma15l2mXNFlmxrzq07+RZwMRHPyba4o8leehb/xr52", - "dzRJjQo+lrqjSWx3yqDujpbT4np0QTHewU/+h01NaiCAcKZJtGjKg+PU8GuogmLZJtj45+1Xzl477y6j", - "A74Nrt2hsncXhip3GZMWNmZt8uKvFKZwb0EFt4cbnytirR3ROvMi1wqMz5D8i/b6JqZ4jTLjVaUKvKbo", - "781rLwXaWy4lLHvrXnJJJxNfWCZScZTtziITLFIiSs5ZViYmgMA95nCyCZWgrbl7qilWYgQIPKcNu0S1", - "XU1UW1dSUyMmN5m6lNHZDqQvlWHZVj3NIq+1CMZR2LmLxindWVXc5OKWoto5578uK3FFj704CpD31FzD", - "RXZweAebCi4ylOCK9ejqtxzo0LKciae0G52pZ+tlkPizZLWVWwpPnuHal/o64ycv2qLipM3toYTq7vGk", - "HXrXTOEFw7uYDW8AWjDiASYgIUZ2HNOv/By77Kdk7rDLSpkhbzBMuM+EAXRJEcp6vkbOfHd43PDmGEOZ", - "OFYKWJlD4AsfTxBxginSSnnu59JrWZTsojsE6aCsGnLh+SyG0uKMkhDoDixNB02FtEoP62HdO3edHBZy", - "+GJceB+4hSQuY7mTxTsni6uMYPXEZGP9Lou3VrvoRIaAIn/Vlu1aH80WJ7WOMuwejd1hhjZyniVH156o", - "mheZah1W+eNLzuSJc672ObhXYgro7eqrUFt4u62lWSJ7Cazz1Ozao22UMdf6UJuVnGhMy+8TAhexKDjB", - "2lq8I/na8vE7CVIXA4MwixAWIoQTQbB7OsYL+wGaGGVbDJ1A2rEmfZfVObDlYda8Y+FdTChO0lBsVUP8", - "NgrjlLlUuX9It9znndBUunTiGvnCNvwlBEq+ptoION7M8qHpz5CM+bCdaHk57aBdoRz9TULse3eh2OkL", - "hdyljUgN4c7be4iSu7qckzwyzOhr7dyseZQrR8V3hlSKkLpiexQZWSQu7+jI7ejsgLtm2FfIf/lqA2IQ", - "Ewu9eQN+gX84Nrb07IZmZr9VrQC5tR3n7p4FX2W8ZYz1XCrXm+fpCcmFd334Xn42vPnDMsdE97rNyldN", - "mUVQTL/kOF7WSSURza+X7YvMqe98aGrNKY9zdBXnlIpzCl5wg5mo8JLKy9Wf08Ft/XCVYkEqEEx3Pd3J", - "unTFParmKdVfUNsInJ/qP5u84wVOaDyBBZm+Zmd5ifX1oKkYfMVqgtiuZVMeO+e5OeGwaJduTjbsFWlq", - "eX4+YC6ORhM1d4RwhlaB3m/g6yEbvWPul2fuPL36Sqkuz2FcxZpdxBHb7s6gvSWD9ncV96FNYnO+SW1V", - "hvVJHDwHMdyQHjFmY3fy5tUoE3zDOo3iF9IopO9rz+I53sJLvEGQed2wRteoY32W0cEd5ANZsbuTAWsH", - "8Bxg4gzPWN27OXQCIHfQVD8BYDL0jQUU3h3rCihsIXKvTaV+VfJ0sTU76rFfQpbYu/PtZCG28kywlnYa", - "TeedyDWFzj+xfhVhneWNsjEtH6tzgDMBxJtX/BN1h/ybf6VONe1zZNjGr4qw6qp1/00/XRd0Do+GUiOc", - "bLbhbMAHbHDbM1Q92JstA11xtF0ujibf7aJzziDJtna//l1+d1s3MXvIZJfNAxeDhCLNcFssgcUbf1eF", - "8Zbg08R6a2ET97LNwtXXhkQ4mACSYmhVe1W2ta5Vp5q6WF+hydoAd4dC3woq1rA1SF9R6DdD85pKv+oL", - "c6MFdMCUAlrxV9ArpQgfVJfgHh8eH+0d0v9dHx6esP/9jwH3onufTqAnXh8QuEehcG1LiVOIJ3AaJXCT", - "IH9kM6wT5hosT1GI8Hx5mGX/reJ5XUCvFdOyuqczFQU811bjU5FV9lU+J0+bK/RZBWhbpT7LuuPyDoru", - "PlHjodiMCYI5JWzKzwNHgEYPuiL7q/XoLX2Pr7kafaeGd2r49tXwTrfsdMsXiTrAK77ewARQ93TDC53v", - "WbyBzAna86I0JA1HPWsjPdcySQjcAxSASQDZQS/HNR70nyHhuRz4lM346oMMmgIMXrnbsLBZS/I6JxVO", - "Ph2vG3i9gKTlwo6K7J9imOADL00SWM/ZvOaxaOjQbtqqx58hORWDbZDuWHXfdnTGIO7SVV8+XdW2uDOl", - "+xK5FYs7V8l4hsg8nRx4IAgmdTX1TyOqcBHYoo73ZzY0K+R9KodvXSfbE/NuolJ2AXdygXW1sY3oW28B", - "dAVxsgL6zlQXb1dMXIOwKJoFcDP0xob+xemNo2/N9JYj7pejN/6usk35msKb0vlrj43HNx1Bfcgau1t6", - "MbvtU0/FBXb6ovWxqj5gjLI9Lr9kY6S9A+B5MCY11VjZ93Zv8vM+7maCufjglWfkDQFYNdTHV77Tj7a/", - "/KNJHNuVvbenrwSywLKaugv0ezv64n3cTVUxoIOvgb74yjv6aqgxSZG0BH0F0QzVFJ09j2bYQaED2Nm4", - "X6NgnLOBNkNL7Aim42+pDpTVPTqIZjPoO6h7r2G3rs/FY51Sje09OYhmUUoamCFKiR03ROnL23oEjUY7", - "lhXdEWmDMsqox5Zs+fP5eI7iFlcgpZPdNYgfId/ybiKoeaMErp+0/X1IRVF3J1rmTqRisJkkY4DxQ5T4", - "ZlnKxaSQpI5sXydSr+SYm9MxTucgnGUT7ZKy4THI/AxRnTh/ReKck1WR0i2YKIEzKsiSuksfb4FrNZKs", - "oOym2EaCsUsMI5HXublehZ4uSchW5+Fv0m7Cw5C/TLubDoYGUdPS41CqAH/wU/zwzFdIh6qu9QxynDaW", - "suYNbcJUZOlzY0BINtGW40HqnleC+R53hZ93sPAzJz+Lws+9jL7smONA4NnmviWbyioQ9RwjjlDrZ5J3", - "lm/WX49doGaZBxWy7erYc0dfVMi3qC2PZrzJ/rApEKsxbnAKsywEK4LN6mIX2RSvtxz7ErGKO6Ze70wd", - "9hZl2HuSdNgjIoB48xqzSS0h81avhpY3cCtlCCicG3XF0+m9Q6Jse/XSLXmNQ9Zxmp7TBEOswmyl06Qc", - "5N98E1KLe1kl7LW4F+1kpPz7NjcjCWBXtW5LVesuDEXqBLEqFLNknHzPqnSuFSe0ULneQsLIkkkiHW+9", - "NG+p2ShrSUDRppvZJJVTuihmj2fMcw8TzOsfGJmxRRL5LvCkJrmXl8hZQ5Wd5Wvs6AGbJVEas1TjHAS5", - "UUZQWKev8KkAzEtIphUzVgXpdUmruyiwsl3ZmOAiCZrN6jyZ17yBA5wQPixXft/+3cydlFzXGnbZd4ZT", - "ZkDDKaUO6PdE3WsCMcl4CmFnCok3h76pIEUu+Hf83i7IQNnVNi/klyosb/cmb/s6UeG50O4lgZcViTv3", - "vJqUgw3vKNi+lWIhmoVswLYvokipYyWW/+CNX9Et71eQyxuWcmJTV1QFO3m3UypgTorLqoDlMJUJBAlM", - "sjCVnjZwBSb3Uh6kSeCeuO7zj+f/HwAA///pYTfLJb4BAA==", + "/8fh0bGWUU303xc4KfIAW5eOKijoAi7oO3RQ7ERTh2IWhgR5TNCpEP/HnQCMPLfnzqJoFkDKixmPV8RY", + "hZlNYA/pCZAAKfZL0iSkAqyGawXlZENQaSg6OVHIJLdCV1VCYuJQixv6hSKED5HDWJXujeJUyFy5mBoZ", + "dpUTaUmUxehLhImBAiNMvkQzp381dOa0lQrjnJAYnxwcCPrfF18oceqOHxCjr/CpeZ47+FSYJp7f3eak", + "CyaeD6fW5DuCOEoTD+rFOJeJft+weoIWUDkUEzGW8wCwEKcFqe0eHx4f7x0d7x29uz4+PDn87eT97/u/", + "//77/7iKmuIDAvfowDoUIYMgQD6nFwWInoNC5+aGCwY6tArIZHJ89P73w//aO37/G9x7/w582APHH/y9", + "90f/9duRf+RNp/+g8y/A4zkMZ5S53/2mASeN/WXREwBMHNF/nTgq0T+ig+e7qIJs4IXr6A7qxMFjjBKI", + "dUv9Poec3SlxEtrdEa33rTd2AQnwASfBhjOiQLFGOXJdkiMZbPvFfT3+8KEJhxlsvUycZMjQItHzYEy4", + "TjCCf6WQC48iPrkCwDG7GlUuUGgm0p77uBeBGO3Ry8EMhnvwkSRgj4AZg+IeBIjui3uSrbiXpsh3nyuE", + "xOHVrfdjGtxxnWtwD0NiXDK8l3cfK/1UM2Sjpspn+KEDCsdRiGEdVFUC4t8oxVhBzGaqArkCdZspUVnq", + "KT1iAwvcD/0i9ltTXn6XTJlgaUOJVntHIWRLYjsnpZF5VZwPh6F++/w0ye+MD3PkzZko4CIKYYdR/767", + "PM9EC0RCFPTkRGxRennU59KIq9wriSM2vo4Py0gzUTyREr6KsQJY9WDwUcxw1NMj8H1E5wXBN4U3SijL", + "2jiS8DP8MWpRgMxn14/FyMFugDudDkb738EnY3cDkriqxkDKyXp8MVY0byOKSBQjr5+YdmoB/jcKHSku", + "nAtKXX/rjy7+Lk+88cXYYWOsQuHZqbBA4X8f9Rbg8b+PP/xWPR4yYM0EwS/k/QAmZLAAKPicRGlsZm3a", + "BOv4KECY0DXyFvLal2DX+k60xPJ9dA97bMbq2gWoTStvUAj44PpjiH6S20rX6pBIWBDWsrdyXT03iQLY", + "dEjx1XyDiwlMRrS9Fh+uGKwJK0Z82Kl13FKzDiywZeAgneknpV/WP2lPWCMp91YJS2ibDCgdHrm+sUXZ", + "upJoXEnJJ5k1r5k08/Y6uxEFd3hW3MqyFVfYeI0LeYiSu2kQPYzScJwuFiB5slINv1e71Wh4/OzIFvJD", + "bvgZ0N3U2xx7zt/+Ob68cCZPBOK/Nx9i2fHFpv+6Gg3IMc6RjuljMENhZpCqQ+hV1jJTdJj8erC/YGTL", + "qaqmEtBdgbIGxMvEh8nHpzOUQE+CBMN0QXcOYM/l3h1FfpT2QvT/JH0fsm9uNDB2HUOQeHPteWui9wou", + "pwBprYxM0Kf0jKGsyls5SRoWbQpml1YMQ5/C0jCwaNZm5L9SmDZDzFu1GTdJw9ACYtGszcg49TwI/Wag", + "s4b2o1M6/Gc00QikOp8ik0uKV1FI4z+jyf6GrEOVMTGBsT0XjgmMdTd79ZypKp1oAaOU6JcvPjYt/R4m", + "GEWhdgbz2ZGBpQ6Qma/40nUaxT+jySjVWP88ZmIIpKnTzqaXdcqc2+YmIwgwJxSNVzZEeN5u6j85Rdbt", + "KCVa3tKweysQXQJxGhBl1BzDmICEtFsMJoCk2GI9VM7ytoK+R2nYjsTp5rencu8OJvUs0Ga5inLVBLJy", + "wJR6Ls8vxUEkgWS7YOaacbZN8gi9GlycDS8+uz13dHNxwf8a35yeDgZngzO3537qD8/ZH6f9i9PBOf1b", + "d9ZSJUTvsbP185e7arZYTMJMR9hsO9qq6pN5I7TaD4W4aErBLwxvEZpGi6cCm5hIR1xsmQHw7r7DyTyK", + "7l58kQos61piNDtHIWzlfqRHKPtM1QcqT+RBGkQzJ0AhbON74jFK2jnocKJBo2pi6s1baG7UJWypfro8", + "cCqb4UeOqnN4DwNV1JwNPt5Q8TK8+HTp9tzv/dGF23MHo9HlSC9TlHEy1d9q/wsQ6ASJ+P7yNydJVnrp", + "wT+ucHsqjtDy/iQ619ygNAhQXTY/XS9NEhiS25jR7nHPDeGj/Ne7nhumC/YP7J4cHdILUZGzCp11zmvR", + "wok5FWYTH1tdORRYtBEe8LE68ju7kfN1aX3uEQGBesGjTZldIkCYcCtdHiF5aHPD0Uisf9Hb3TdIEuRp", + "5HGYLq7srp+MjuUldN+03n9Z3Tj5WIi74tn10zjgyO6qyUcUF859PWoK9soM1MIsPRUhOvk/AgSeowXS", + "CAwri2NCxX9AB9CK6ABgMoJTFBjM6ywUQ8RqqIOxOI2EdYTMTbjGQBY2wR8gSA3HzgI8okW6UO/93FCO", + "HRbzJwyVYrcfUOhHD/rtXocltAHB9+Z1SCmiWccC+NB2Efybfgr+jS2D7iEKFVdvjmYenTaNEg/6tn47", + "5Vag7JdcbwZVgcJ+qPS8A4dgzlvaYzD7vMJBWB6jchRybEqsKajUjgY9GJKxcnsteTcYeCZ65l8dnVtf", + "NTe0uY8uY39YwXawMQOBQGluIahcl8vhHfU8km1ET71JC1jKo2vFPqR/vZ04qRGMA/D0S8Xp8CUpZhhs", + "XFmBHl52fUrzD4eHDestwW1atclgonS3F9olu5YtfBK6hHI5Y/YatopTnYGyQs20GR21ZNvQDDiDmNwk", + "Bh3rZnTukMjBMPRZYIq43mKHRJtxFZsOiDREf1FtwIchQVMEk0yLFAqQiNfl8TNqePsEBlE4kxA3yMre", + "JsN37AyZtSE5Y7gA8TxK4DiIyJpP2cIJpnd8cLUSBxFXskUPe5PNkieesImblkU/0+uOWFjzFqvG7eaF", + "oiCQXh/7lVYOaY0CLJrYg16inxwtPfVUL1vCpQWcko9qBKya7eYgDGFggld8dpCvv21gOrjzwEfX63F8", + "hAtjiJCcgoUKLTnJSiIILEyrp99WWDrtbl43G3yVRe+E8LQTbxIRGbqLdNFTyFArAQmMTXJP76uco8BP", + "YNHx0qA7bci/GIOkEkrfCEkCgQ8mATRtrvyeJdNwgdhIJiu5vQ0zmClAWUWBHKSbTmwgt0DWbP0G3Nx9", + "MoijgjVXsWCsyRnOiPC7SadspIFCd3wapSHRgwuNUC5zHc771GCorJIWvPkWzmARu5C1Xz/bRSkxgbgk", + "RzIzbX9KYGKPzLUHF/AuNTuzgrZlG1dD25rEiYWsabPirEvNiqnqY4hpsDqcMgrMVlYbQCBQ10+8ObqH", + "r1IuqZ5aO/h2SsREiQ8Tfacark8gSZ5qpOjG+FG5xmyHJWpuDAoSJB57Wu+nid53wEpeYkCtqVy0MQTd", + "e2YqkKdNBQ3I13dQAhI0JCd50GI9wtbIelC6gfcwQeSpTe+x7GNFd59QgskYciXZnvbOQdteLUO9+C2j", + "AGBp5gyzCprUKAy+vzXEvCvx4gUybSTkXKRLZ81o8K+bwc3g7Pbi8vb75ejrYOT28h9H/evB7fnw2/Da", + "7bnj0y+Ds5vz4cXn2+vht8HZ7eUN/bk/Hg8/X7DAtvF1f3TNY92GF8Pxl2LY22hwPfo3D4vLI+B6Lh3r", + "8ub6djT4NBqIPqOBMqo62fj8krY8H/TH2ZjDwdntx3/f3owZ7HQRn84vv9+Obi5uP48ub65uvw7+fasG", + "4hmaCEB1XiotiyhYVOJwxAJHw+vhaf+8brS6CELx1y1Hw7fBRQnT1hGG2d+0tQ6YvHBOuaQPTETa18CQ", + "nPddlgaJHNZamgUWrBfe19YBASEIngjy8GVMLlNSM2puZ5gD7EQxgb4j7pLZIPo5Nl5ewJQStnJOWXMx", + "AmN6mDbhcruZlhvKfTcnXGrXvANSWb8XusTUWbTHSc4d0QmYxFZ6o3A2hoT+B2+PRXn5nMFjjOgus5hc", + "Bkz9+LwXnwY7D6xKCAsvdkACHRDHSQS8OQpnvFwIQ3Dd/DJhlBMJizhYEgq+ZFmPpQoPC1GoxYVigvkE", + "UJAm0AIU5v1SAVEt95glOennDADmSzV7VfIgJhCKnWWeFZGHZxm2AB4lkX1ixonQezLGJzlT2cQBRMbc", + "CKpar0HdLAm0AJvlwjALJthM7vVzVhqm1iMkCwKJInDbLJazXIJ3k19AMJTJqyE/m7HGW9T5NdgIhRIa", + "S5yYhcz0fK/ULNsG2tmZo0SQcrsThO9pFf4XIyj7hG7Kek2tbzBMeI+rdBIgr44U2Hg1NQpUmHdm08X+", + "LbPpI7FP8mZx+f2C3Y76Z9+GF27P/Tb49nEwqrkQ1Ic8M0N2qfrAVbGcSTUytYxzFrvdhIkCHIoloG7u", + "NuOVA2IyBEjKV7GYXZAHf/AbmXqTZLe+y4v8ojyoQW9BrdFpdiBZ1MQLs+8OC7HUy2Ae0Uwi5wEkLN+2", + "ou/w3vr423Yh1Pro6fUERvOxzUvUw79aLme27c0cmhGJXVh004a1j4ZeQAITGRMtj0o+lvM3tA/3nSPH", + "B08958h5gPCO/ncRhWT+9yXd8Bl6tDHSZskqEXUVBcjT1AXgKnjdrTQrp8ibavSCFpK1yH5NMXcCOO3q", + "EjSbwUS5H7SsgVW1ErcN1bthJR7fYrkldeUNccVrqXRk1B5UQMz7/4oNap1F4GUtAhu8qW+k/KG1vfTZ", + "yE3fmU/eHNGMr0CKdWlyKrlzx76DsBOz1g4IfccDYRgRB7C6rawAvCx/Uka8Fjqsu1I1mhSA7ycQY9W0", + "UNCS5F21amGgH74APNdJ6znAc3XI/4dL0wn5zRUNXj99zEuRO6dzQIwT/gETNEVN6GUGEipL7kVzUcO/", + "AIOeoucAm18K0M4BsqcBHAzJFg3/PsJxAJ4KBC33r7UtoojdHwYCKz6lYC6aBx/MSGQ8CB9yrEmNSQ/7", + "Esd29lTDMwuJqgMkA6IWf6vBUClGkD0koeLJhPLzaIbC5cs1LsffK1Vv3DmMyzXGTbgewRnCpEa67yK6", + "7U46g2DYwd2Sxc1tN01Vj/Ecxfi12skqdsMtnuabOGX4ZLptEykbXJVaqx3YjhlE6oFQw7RskZpSyGTf", + "NAmWcZPTcRtRwivTrliT1mKRGHoJNHjy+LeswIHgYXoTcoZT9pJPnET3yId+zwFOAkI/WshOLMdoAp0Z", + "DGECREqVmg55vDGMt0ezv5sEuNzebJuUMzgbkU2l8o4U9CqKH6unigpdjIwpwk1vATHWwIXsqpeX+eBD", + "qU/VtHLAziO/1WoF6N94zyyo/lT7wB8F+cv19ZXDG7GX/SQFJwL5FvVYFKxkMBcm/mGJ8HoSkhU9TAZ7", + "bj+UNC9bWxtotRSwNO18y7ZO+nA+D67dnnt1OWb/ublmNlTTCcnTi3BdWizm9nthafBA6MQwoXS13ypu", + "CtwDFIBJAGWWT0NR2Oq08BF6KYGOF4XC3xA86R0KVNVghciTYcMDS8xZgWYh9J280zqeWloxpT8AExjg", + "emcLa8NYKj8OsmPAOi8fJud0HN2WBQCTLxAkZAKBRVaw2CrmO8MUQODMZe91lxsCnImpWjDABEwCliyx", + "QxAuwKOZ0DXVkFYj+M3rGWb9IqkUuNHlYtI2WUJ67txqSbClYjoamsUy/1+n7IgaATx/nTPOkoCU6g3o", + "AMmCn3WJ+ewYrOBWivD+6fXwjwErf5j9edW/GRuCt/kP+QkwHpx/+nI55oHq3/oX/c88Knzw8cvl5Vft", + "EOI0M6bki8NOPEtZhLqxroDofdOkPt6MzjXDt9UmWXutJqBIu3aF2WQpTtp13bn1NV517k1vmLz+wYUa", + "PLy8ccOoNwsgZRjM6oXYpRtKu3n2nhbmycp8LXoL/Z/RxMD19IsOICtcifLea4tGbyP4jZiTdjnNmQtm", + "y69V7v010GqSogJ8+/JDf/COWVWLupCIslwxMRId91Sez7rAjxkkyvcsZ6Hk6AplDRruzZxBghnuvLyr", + "M6N9MwGp+Gf3jYFHY5IAAmdPpmOIf3VIxH1o8uFLdVYeoMSe+wTeXBSOFAcNz7y5HV7cXo0uP48G47Hb", + "c89Gl1e3F4PvA3YBYXlW+T95MtLo8ubi7HZ0+XF4oT2NWipPuX5U9DeXH3x4d9x8x5RTlxHY025kHVUM", + "z3QOzQzA4Zl222Tvrygs3Oo+3VycXg9Z9N3Zzaj/8ZzqBGd9fcqTOoiU8604hc2uYT35XX94rFQBZMvn", + "Djtb7G7dorUxVpTx5VeYJ05rxGGpim+Vre/gE9br9nJ4SpY1U5TuElRMAAfH0ENT5OWTOH+LAcbQd+4R", + "cKYoIDD5u2WR4O/Fhwxsgr/0AayFp9iVlw65g+DC9MJJFpahltg7OlQq3m2s4EcWs9ZqQbxqgj1d5hU/", + "1njM80oeXLHfthWDzz1W06y3DcLGyo+qj0FlJULqa3vwUErof3xqMfi10ksJDxe6TEvVRzPC6i+L/KG8", + "yJOVS1UX+6NemOzITaSuQGUd+HWVhvvjU3pMD8anted0PkpN2X2VlgtSTJGMDZOM5yCGnezuZHcnu19S", + "dhvm+AVF+1rekBKXyLNG6cYmW+q+UyQEw6WntKEap20UXikcqykqFYVjeodOA72UFLWoN1Ne8fuSr4k1", + "bDHmz/UvUyd7k2W9y2WuGxZhvNyxOjlt6EgOdco7NmkPpeaV+QU/aBMOJS9pPwqe0X6TrKf9mHOjvm6W", + "cTXXYKbDX8DP8tXttCsbLPURPhzCOgIRXH+aUA1zqmf8miqKt8jAbk0TiopGU0NZ/Vvhs1j3tFi/wvba", + "dAlvGtHK1rH0wBl+1qt18XNQj778aLwVZuj2aObpH2tI/Gh2R9SBoagZZZYtmLNtNkS1gFNtH05BGpCr", + "BEWykJSO/VkjJxatdAzcaL3N/S0v5EXJCi1agIrF2X+dFxTWKLDIuzPa6+m33Gxv5aJReLoFa2HF0WJw", + "TvKPVkCoOe22htlaZdmsxEqY89KNykA/mtmB7es6LdttCORNIZx7f3OTdukd9QSy2IqaaqQL8NjQomVV", + "RVNNRB5Em1IhRdX3BYdwAkECk35KWOIawyiTveznfFPmhLDCWF4U3SEomyO6q/wn6WE8cecs/EzJWQMx", + "Yu/5PzMzxjQyRFbybk7/asiqQBJ2RS/+mlGWe7R/uH/ICDOGIYiRe+K+2z/aP2QZEmTOlnYAYnQQiNK9", + "M13w8GfpoKStQoixk10P6S4C+dqGey6+f2brksGebJbjw8PqwF8gCMicSeUPuu8XEcnmLOyMe/KfHz0X", + "y3f5KYR5Q+mq/o8Y35tD7879QfuztSYQ+E/Ni6XNUN1qR7LBOpfLgGMJrjyhkyRgOhXVYOpWn0HbuPz7", + "owMgsm/3WLLFHvMX4YOf7Gf1t2cOYwCJRhc/Y79jB2Sv77Ekb55SwrpXMFZK6OcjMFpMAKv9QMGuKaFU", + "mcFhV0nGX5Sec+6qLMVVuZ+bAblcXPlu+vyjsvfvq9gap54HMZ6mQfDkcJT6hacLK8h77rnvOZV4UUhE", + "3V4QxwHyGEYP/hTFT/N1NJxWrEq2SBsqu6oXIKBYgL4TJc4E+DLUmYPxbu1g6KD4FCUT5PuQ67I5fXM6", + "qSMzSfGi5NKPnvu4l+XD0w+iYlNPQxg/2CWKeJqUZK68r0LifIRfg8QZPXyMuOxcCzFYFPvQkEkttkjk", + "pBLnRWw860X0WhZiqJBZhb0gBjignRiwFAOcWjYnBtQDMkZ7vLjHwc/sb3YaxhHWKA0jeB/dseqV/ash", + "LwsigjKyGUtiIkas7og0D9DuNlIiG94gEySsO3XcJWx5gs4ZdL82UeM2VC1Ih27stdg5Scb5b3WUnG15", + "gYK9IEr9A/Uqa9Z2Zass3FBeJ9ggDgoxASGrklUk4lP6WXqRzUrw5nHLAHHSMEs72hkCa9DaOYJVt5zY", + "+m+KQ+ZxTw6xF8Xcpy1ONGW/uXH14Cf773PdflMpxVrtVzaU2Vj5RjZKIjaEUTlhX7cqhNa32eL1gYbD", + "O4EkQfBeiDWODbZjnWwrkLiCmZy8OYprpBqnnx9mCj9oEmtsWzKp1kDzZ5kAe+t0f8ZIuKP93aL9BVz6", + "DDee3ts7uEUh9DY0lR2Jr+QgX8cRTsc4UN4dxcYdP0eYXoACp9DatMG09bDYcGO7TecSO66+uN1u82XW", + "fWF1u0QI2dazjShtQnX/1U1mr9ge/GT/sTCvOmP11dvKFqtPGdtbUwtjGo8yBuJOmk2LONmlM+doO2Dc", + "hCAl8yhB/wt9PvGH7UzMC1CwOj4gCKIH6OtNtWWqlTzBfq87+zjRFTkmxAc/cYituKX4cnOVX0Lcgk1K", + "z0AbGUWI1J1jkxIyOkbZQUapEGzGKhfjWkYJsYZN+Odn1QygN1jSeeVdpcIirZ0WJs7IoN0Uc/TMN7Q7", + "+LTsFU2B4fjDhwIQR9ZXshoGjZOI/gP63Rm2Q6xp0u4RmacTB8SxpPbqscbblPiRwHgvSdnhJf58PgD8", + "jdYmzV60knmdopBKlVV5vgbTueXAFkwrxzMfaALebTOuyGolkYPvUCxh+yuFyVMOXDSdYnZj1YCCQvLb", + "e22Ca/10PO198mSYkn1uOeMmDTWaF4iXsNjgN26tobO+386sBa57AJgJn2mUhr7uPllgf4X5M82A/jRK", + "a71CGQs3y6Q8LNsskXibFvJowAftpNGbkUb5E9KdLPp1ZJHC+JuXREE0q5dD2AmimROgsKIbVf0659Hs", + "HIX8dOzE0G6IoZ75haYA3sMA03l5nZKaiVnLwsy1FmlBB7QXT7g3rBxDevA6bDYFjmmUGADhHdoCMua9", + "NEB8Z2+rRg4LrTevP1KLB7ScvFB4wIAHPr2fVTioheJMabYMJHn/zR5SqjRoOp8oSXaHk8GtyU6FTAor", + "Z8F5NGt/DPDP2Gyn4sXWsQPY4y+GYDoe7sebupuJVOWDF99Eqw9NJZEozf0CgaiNJC4KjSiRp12caUbi", + "fK9zYmuKKtVRdGaK5dVtaqLLWWjKI8IEhbN6An89ZtkthIvbMWGeZvaigeEdP64t7rtFlHctX+pzoOpj", + "bECmrZpi0HFTPojtdWQnOHibyRJLWA7Mm9DxTkFdq6NWe2bqtVDR2idKZdrbWz3cVA1zfblQ1iro0Qvn", + "QlVPwC4XylZHXSkXyu6UPMCQ0P/i5rxp2cWRXeozoRRyQeFsLPpYBmO/kWNSQcwKZ6S6Jx0rFcJ3jWha", + "Gx9lCYX1jrYsvw/b5Q92+mQWc8zwgfOqsK34RBYP6Wx9ZeUxS0LE7TITmxTGJZJlOx2RIUDSuqIWbtKE", + "UZ6046918ZdghCVTf+sPHIuoDsxSSAqhHby3IUnutZw1b9mNegefrJyotF1hVquKeowMWJGqakFWM0zK", + "KyhWsOWyojWAynMsy4GYpKEo9wStYJVtrd2f+hLGL+SSZvv5Mg5pNvUOuKNVOFRndA2xZKmWd/BJPNsW", + "A5RU6CWroP4fym5HJ6zpEX+r7Zj/65iKd916NFX6tczQWCfZvAyZyGxF56JYtYEl11vbeeM5zl0UwFpu", + "BlDGeFpmNtuakOsS9bsrAEOAKIZcaxbm/P0yYQh2JTRUmy/PynnzUaDH/9jOrLJwrVBP4aMHoV9JUhMX", + "FJkxZc3nzReTg0ka3JnDfj6mwZ0gD5zLBFwrFGifNywY6PJbCgf8QtKhAqqlSaEiL7qw8R0TGIxvVamB", + "1yw2PPb+SE28IPvOLRvK04gFndckRnicCR/hLWsYDAH2Goa4QSQwDsDT2uXIi70vUy4L3iCaGNKgX3zw", + "vhNSuyikRoxSNyOfmF3N0ujKjXUWhtev8Knz8+XWx6Wu7wzZ3RVed4V3hDF4nXwgToOagrn0O253NI/k", + "EfNWj2aOgF05mtdjZ+PAdVr9WzswUXiPCGwbcS176aPIhuxrd1bK4DEFH0uFjUlsd8FiunjqnBY3FETN", + "J6il9c4eroRNc5TYRUtz3L5oiDQHd5nIaEEYHVvqw6EzvllP7Kbgc/nDHv93u7eRLFi59WtIuxVgU+Sr", + "etj2MnS89rO1kXs1Tz3tGPfqyhJm+2NK5y7uY5snlCw44ZXXH9xBTthsLu5y5+6LZeNacq7mdaZd5lyR", + "Jduac+tOvgVcTMRzsi3uaLKXnsW/sa/dHU1So4KPpe5oEtudMqi7o+W0uB5dUIx38JP/YVOTGgggnGkS", + "LZry4Dg1/BqqoFi2CTb+efuVs9fOu8vogG+Da3eo7N2FocpdxqSFjVmbvPgrhSncW1DB7eHG54pYa0e0", + "zrzItQLjMyT/or2+iSleo8x4VakCryn6e/PaS4H2lksJy966l1zSycQXlolUHGW7s8gEi5SIknOWlYkJ", + "IHCPOZxsQiVoa+6eaoqVGAECz2nDLlFtVxPV1pXU1IjJTaYuZXS2A+lLZVi2VU+zyGstgnEUdu6icUp3", + "VhU3ubilqHbO+a/LSlzRYy+OAuQ9NddwkR0c3sGmgosMJbhiPbr6LQc6tCxn4intRmfq2XoZJP4sWW3l", + "lsKTZ7j2pb7O+MmLtqg4aXN7KKG6ezxph941U3jB8C5mwxuAFox4gAlIiJEdx/QrP8cu+ymZO+yyUmbI", + "GwwT7jNhAF1ShLKer5Ez3x0eN7w5xlAmjpUCVuYQ+MLHE0ScYIq0Up77ufRaFiW76A5BOiirhlx4Pouh", + "tDijJAS6A0vTQVMhrdLDelj3zl0nh4UcvhgX3gduIYnLWO5k8c7J4iojWD0x2Vi/y+Kt1S46kSGgyF+1", + "ZbvWR7PFSa2jDLtHY3eYoY2cZ8nRtSeq5kWmWodV/viSM3ninKt9Du6VmAJ6u/oq1BbebmtplsheAus8", + "Nbv2aBtlzLU+1GYlJxrT8vuEwEUsCk6wthbvSL62fPxOgtTFwCDMIoSFCOFEEOyejvHCfoAmRtkWQyeQ", + "dqxJ32V1Dmx5mDXvWHgXE4qTNBRb1RC/jcI4ZS5V7h/SLfd5JzSVLp24Rr6wDX8JgZKvqTYCjjezfGj6", + "MyRjPmwnWl5OO2hXKEd/kxD73l0odvpCIXdpI1JDuPP2HqLkri7nJI8MM/paOzdrHuXKUfGdIZUipK7Y", + "HkVGFonLOzpyOzo74K4Z9hXyX77agBjExEJv3oBf4B+OjS09u6GZ2W9VK0Bubce5u2fBVxlvGWM9l8r1", + "5nl6QnLhXR++l58Nb/6wzDHRvW6z8lVTZhEU0y85jpd1UklE8+tl+yJz6jsfmlpzyuMcXcU5peKcghfc", + "YCYqvKTycvXndHBbP1ylWJAKBNNdT3eyLl1xj6p5SvUX1DYC56f6zybveIETGk9gQaav2VleYn09aCoG", + "X7GaILZr2ZTHznluTjgs2qWbkw17RZpanp8PmIuj0UTNHSGcoVWg9xv4eshG75j75Zk7T6++UqrLcxhX", + "sWYXccS2uzNob8mg/V3FfWiT2JxvUluVYX0SB89BDDekR4zZ2J28eTXKBN+wTqP4hTQK6fvas3iOt/AS", + "bxBkXjes0TXqWJ9ldHAH+UBW7O5kwNoBPAeYOMMzVvduDp0AyB001U8AmAx9YwGFd8e6AgpbiNxrU6lf", + "lTxdbM2OeuyXkCX27nw7WYitPBOspZ1G03knck2h80+sX0VYZ3mjbEzLx+oc4EwA8eYV/0TdIf/mX6lT", + "TfscGbbxqyKsumrdf9NP1wWdw6Oh1Agnm204G/ABG9z2DFUP9mbLQFccbZeLo8l3u+icM0iyrd2vf5ff", + "3dZNzB4y2WXzwMUgoUgz3BZLYPHG31VhvCX4NLHeWtjEvWyzcPW1IREOJoCkGFrVXpVtrWvVqaYu1ldo", + "sjbA3aHQt4KKNWwN0lcU+s3QvKbSr/rC3GgBHTClgFb8FfRKKcIH1SW4x4fHR3uH9H/Xh4cn7H//Y8C9", + "6N6nE+iJ1wcE7lEoXNtS4hTiCZxGCdwkyB/ZDOuEuQbLUxQiPF8eZtl/q3heF9BrxbSs7ulMRQHPtdX4", + "VGSVfZXPydPmCn1WAdpWqc+y7ri8g6K7T9R4KDZjgmBOCZvy88ARoNGDrsj+aj16S9/ja65G36nhnRq+", + "fTW80y073fJFog7wiq83MAHUPd3wQud7Fm8gc4L2vCgNScNRz9pIz7VMEgL3AAVgEkB20MtxjQf9Z0h4", + "Lgc+ZTO++iCDpgCDV+42LGzWkrzOSYWTT8frBl4vIGm5sKMi+6cYJvjAS5ME1nM2r3ksGjq0m7bq8WdI", + "TsVgG6Q7Vt23HZ0xiLt01ZdPV7Ut7kzpvkRuxeLOVTKeITJPJwceCIJJXU3904gqXAS2qOP9mQ3NCnmf", + "yuFb18n2xLybqJRdwJ1cYF1tbCP61lsAXUGcrIC+M9XF2xUT1yAsimYB3Ay9saF/cXrj6FszveWI++Xo", + "jb+rbFO+pvCmdP7aY+PxTUdQH7LG7pZezG771FNxgZ2+aH2sqg8Yo2yPyy/ZGGnvAHgejElNNVb2vd2b", + "/LyPu5lgLj545Rl5QwBWDfXxle/0o+0v/2gSx3Zl7+3pK4EssKym7gL93o6+eB93U1UM6OBroC++8o6+", + "GmpMUiQtQV9BNEM1RWfPoxl2UOgAdjbu1ygY52ygzdASO4Lp+FuqA2V1jw6i2Qz6Durea9it63PxWKdU", + "Y3tPDqJZlJIGZohSYscNUfryth5Bo9GOZUV3RNqgjDLqsSVb/nw+nqO4xRVI6WR3DeJHyLe8mwhq3iiB", + "6ydtfx9SUdTdiZa5E6kYbCbJGGD8ECW+WZZyMSkkqSPb14nUKznm5nSM0zkIZ9lEu6RseAwyP0NUJ85f", + "kTjnZFWkdAsmSuCMCrKk7tLHW+BajSQrKLsptpFg7BLDSOR1bq5XoadLErLVefibtJvwMOQv0+6mg6FB", + "1LT0OJQqwB/8FD888xXSoaprPYMcp42lrHlDmzAVWfrcGBCSTbTleJC655Vgvsdd4ecdLPzMyc+i8HMv", + "oy875jgQeLa5b8mmsgpEPceII9T6meSd5Zv112MXqFnmQYVsuzr23NEXFfItasujGW+yP2wKxGqMG5zC", + "LAvBimCzuthFNsXrLce+RKzijqnXO1OHvUUZ9p4kHfaICCDevMZsUkvIvNWroeUN3EoZAgrnRl3xdHrv", + "kCjbXr10S17jkHWcpuc0wRCrMFvpNCkH+TffhNTiXlYJey3uRTsZKf++zc1IAthVrdtS1boLQ5E6QawK", + "xSwZJ9+zKp1rxQktVK63kDCyZJJIx1svzVtqNsoqjGWj9tlzVzs9cCcYbHNV6+TrQnYV67jWVeSybSuH", + "VhKhrB528sCoIK7GnA1qolXFB7pJxdIOGePdwwTz4iTGk7JFhYdd4GdN5j2vX7WGEljLF8DSAzZLojRm", + "dQByEORGGUFhnb7CpwIwL6E2rJhOLkivyyjfRW0i25WNCS6SoNmsLszgmjdwgBPCh+XexrB/1HYnJde1", + "hl32neGUWbdxSqkD+j1RlJ5ATDKeQtiZQuLNoW+qFpML/h1XpAQZKLtqq0xVXs14CU3K5umwwlu+3TMf", + "LysSd+7tQykHGx45sX3IyEI0C9mAbZ8rklLHSiz/wRu/IhPMryCXNyzlxKauqAp28m6nVMCcFJdVAcsx", + "ZBMIEphkMWQ9bVQZTO6lPEiTwD1x3ecfz/8/AAD//0G8mqSXwgEA", } // GetSwagger returns the content of the embedded swagger specification file diff --git a/api/v1/server/oas/transformers/workflow.go b/api/v1/server/oas/transformers/workflow.go index 5b46bf179..62e895f96 100644 --- a/api/v1/server/oas/transformers/workflow.go +++ b/api/v1/server/oas/transformers/workflow.go @@ -25,6 +25,8 @@ func ToWorkflow( Name: workflow.Name, } + res.IsPaused = &workflow.IsPaused.Bool + res.Description = &workflow.Description.String if version != nil { @@ -44,7 +46,7 @@ func ToWorkflowVersionMeta(version *dbsqlc.WorkflowVersion, workflow *dbsqlc.Wor version.UpdatedAt.Time, ), WorkflowId: sqlchelpers.UUIDToStr(version.WorkflowId), - Order: int32(version.Order), + Order: int32(version.Order), // nolint: gosec Version: version.Version.String, } @@ -73,7 +75,7 @@ func ToWorkflowVersion( version.UpdatedAt.Time, ), // WorkflowId: sqlchelpers.UUIDToStr(version.WorkflowId), - Order: int32(version.Order), + Order: int32(version.Order), // nolint: gosec Version: version.Version.String, ScheduleTimeout: &version.ScheduleTimeout, DefaultPriority: &version.DefaultPriority.Int32, @@ -290,6 +292,7 @@ func ToWorkflowFromSQLC(row *dbsqlc.Workflow) *gen.Workflow { Metadata: *toAPIMetadata(pgUUIDToStr(row.ID), row.CreatedAt.Time, row.UpdatedAt.Time), Name: row.Name, Description: &row.Description.String, + IsPaused: &row.IsPaused.Bool, } return res diff --git a/frontend/app/src/components/ui/use-toast.ts b/frontend/app/src/components/hooks/use-toast.ts similarity index 92% rename from frontend/app/src/components/ui/use-toast.ts rename to frontend/app/src/components/hooks/use-toast.ts index e494ec350..25a2d8488 100644 --- a/frontend/app/src/components/ui/use-toast.ts +++ b/frontend/app/src/components/hooks/use-toast.ts @@ -1,4 +1,3 @@ -// Inspired by react-hot-toast library import * as React from 'react'; import type { ToastActionElement, ToastProps } from '@/components/ui/toast'; @@ -6,20 +5,11 @@ import type { ToastActionElement, ToastProps } from '@/components/ui/toast'; const TOAST_LIMIT = 1; const TOAST_REMOVE_DELAY = 1000000; -// Add this type definition -export type ToastPosition = - | 'top-left' - | 'top-right' - | 'bottom-left' - | 'bottom-right'; - -// Modify the existing ToasterToast type type ToasterToast = ToastProps & { id: string; title?: React.ReactNode; description?: React.ReactNode; action?: ToastActionElement; - position?: ToastPosition; }; const actionTypes = { @@ -144,10 +134,9 @@ function dispatch(action: Action) { }); } -// Modify the Toast type type Toast = Omit; -function toast({ position = 'bottom-right', ...props }: Toast) { +function toast({ ...props }: Toast) { const id = genId(); const update = (props: ToasterToast) => @@ -162,7 +151,6 @@ function toast({ position = 'bottom-right', ...props }: Toast) { toast: { ...props, id, - position, open: true, onOpenChange: (open) => { if (!open) { diff --git a/frontend/app/src/components/ui/toast.tsx b/frontend/app/src/components/ui/toast.tsx index 8c45d1dc9..12d175efe 100644 --- a/frontend/app/src/components/ui/toast.tsx +++ b/frontend/app/src/components/ui/toast.tsx @@ -2,8 +2,8 @@ import * as React from 'react'; import { Cross2Icon } from '@radix-ui/react-icons'; import * as ToastPrimitives from '@radix-ui/react-toast'; import { cva, type VariantProps } from 'class-variance-authority'; + import { cn } from '@/lib/utils'; -import { ToastPosition } from './use-toast'; const ToastProvider = ToastPrimitives.Provider; @@ -14,7 +14,7 @@ const ToastViewport = React.forwardRef< , React.ComponentPropsWithoutRef & - VariantProps & { - position?: ToastPosition; - } ->(({ className, variant, position = 'bottom-right', ...props }, ref) => { + VariantProps +>(({ className, variant, ...props }, ref) => { return ( ); diff --git a/frontend/app/src/components/ui/toaster.tsx b/frontend/app/src/components/ui/toaster.tsx index 544056ecc..613c54a68 100644 --- a/frontend/app/src/components/ui/toaster.tsx +++ b/frontend/app/src/components/ui/toaster.tsx @@ -1,3 +1,4 @@ +import { useToast } from '@/components/hooks/use-toast'; import { Toast, ToastClose, @@ -6,7 +7,6 @@ import { ToastTitle, ToastViewport, } from '@/components/ui/toast'; -import { useToast } from '@/components/ui/use-toast'; export function Toaster() { const { toasts } = useToast(); diff --git a/frontend/app/src/hooks/use-toast.ts b/frontend/app/src/hooks/use-toast.ts new file mode 100644 index 000000000..25a2d8488 --- /dev/null +++ b/frontend/app/src/hooks/use-toast.ts @@ -0,0 +1,190 @@ +import * as React from 'react'; + +import type { ToastActionElement, ToastProps } from '@/components/ui/toast'; + +const TOAST_LIMIT = 1; +const TOAST_REMOVE_DELAY = 1000000; + +type ToasterToast = ToastProps & { + id: string; + title?: React.ReactNode; + description?: React.ReactNode; + action?: ToastActionElement; +}; + +const actionTypes = { + ADD_TOAST: 'ADD_TOAST', + UPDATE_TOAST: 'UPDATE_TOAST', + DISMISS_TOAST: 'DISMISS_TOAST', + REMOVE_TOAST: 'REMOVE_TOAST', +} as const; + +let count = 0; + +function genId() { + count = (count + 1) % Number.MAX_SAFE_INTEGER; + return count.toString(); +} + +type ActionType = typeof actionTypes; + +type Action = + | { + type: ActionType['ADD_TOAST']; + toast: ToasterToast; + } + | { + type: ActionType['UPDATE_TOAST']; + toast: Partial; + } + | { + type: ActionType['DISMISS_TOAST']; + toastId?: ToasterToast['id']; + } + | { + type: ActionType['REMOVE_TOAST']; + toastId?: ToasterToast['id']; + }; + +interface State { + toasts: ToasterToast[]; +} + +const toastTimeouts = new Map>(); + +const addToRemoveQueue = (toastId: string) => { + if (toastTimeouts.has(toastId)) { + return; + } + + const timeout = setTimeout(() => { + toastTimeouts.delete(toastId); + dispatch({ + type: 'REMOVE_TOAST', + toastId: toastId, + }); + }, TOAST_REMOVE_DELAY); + + toastTimeouts.set(toastId, timeout); +}; + +export const reducer = (state: State, action: Action): State => { + switch (action.type) { + case 'ADD_TOAST': + return { + ...state, + toasts: [action.toast, ...state.toasts].slice(0, TOAST_LIMIT), + }; + + case 'UPDATE_TOAST': + return { + ...state, + toasts: state.toasts.map((t) => + t.id === action.toast.id ? { ...t, ...action.toast } : t, + ), + }; + + case 'DISMISS_TOAST': { + const { toastId } = action; + + // ! Side effects ! - This could be extracted into a dismissToast() action, + // but I'll keep it here for simplicity + if (toastId) { + addToRemoveQueue(toastId); + } else { + state.toasts.forEach((toast) => { + addToRemoveQueue(toast.id); + }); + } + + return { + ...state, + toasts: state.toasts.map((t) => + t.id === toastId || toastId === undefined + ? { + ...t, + open: false, + } + : t, + ), + }; + } + case 'REMOVE_TOAST': + if (action.toastId === undefined) { + return { + ...state, + toasts: [], + }; + } + return { + ...state, + toasts: state.toasts.filter((t) => t.id !== action.toastId), + }; + } +}; + +const listeners: Array<(state: State) => void> = []; + +let memoryState: State = { toasts: [] }; + +function dispatch(action: Action) { + memoryState = reducer(memoryState, action); + listeners.forEach((listener) => { + listener(memoryState); + }); +} + +type Toast = Omit; + +function toast({ ...props }: Toast) { + const id = genId(); + + const update = (props: ToasterToast) => + dispatch({ + type: 'UPDATE_TOAST', + toast: { ...props, id }, + }); + const dismiss = () => dispatch({ type: 'DISMISS_TOAST', toastId: id }); + + dispatch({ + type: 'ADD_TOAST', + toast: { + ...props, + id, + open: true, + onOpenChange: (open) => { + if (!open) { + dismiss(); + } + }, + }, + }); + + return { + id: id, + dismiss, + update, + }; +} + +function useToast() { + const [state, setState] = React.useState(memoryState); + + React.useEffect(() => { + listeners.push(setState); + return () => { + const index = listeners.indexOf(setState); + if (index > -1) { + listeners.splice(index, 1); + } + }; + }, [state]); + + return { + ...state, + toast, + dismiss: (toastId?: string) => dispatch({ type: 'DISMISS_TOAST', toastId }), + }; +} + +export { useToast, toast }; diff --git a/frontend/app/src/lib/api/generated/Api.ts b/frontend/app/src/lib/api/generated/Api.ts index da79e82da..7e849bf65 100644 --- a/frontend/app/src/lib/api/generated/Api.ts +++ b/frontend/app/src/lib/api/generated/Api.ts @@ -93,6 +93,7 @@ import { WorkflowRunsMetrics, WorkflowRunStatus, WorkflowRunStatusList, + WorkflowUpdateRequest, WorkflowVersion, WorkflowWorkersCount, } from './data-contracts'; @@ -1153,6 +1154,25 @@ export class Api extends HttpClient + this.request({ + path: `/api/v1/workflows/${workflow}`, + method: 'PATCH', + body: data, + secure: true, + type: ContentType.Json, + format: 'json', + ...params, + }); /** * @description Get a workflow version for a tenant * diff --git a/frontend/app/src/lib/api/generated/data-contracts.ts b/frontend/app/src/lib/api/generated/data-contracts.ts index 3ac761202..e34640228 100644 --- a/frontend/app/src/lib/api/generated/data-contracts.ts +++ b/frontend/app/src/lib/api/generated/data-contracts.ts @@ -549,6 +549,8 @@ export interface Workflow { name: string; /** The description of the workflow. */ description?: string; + /** Whether the workflow is paused. */ + isPaused?: boolean; versions?: WorkflowVersionMeta[]; /** The tags of the workflow. */ tags?: WorkflowTag[]; @@ -556,6 +558,11 @@ export interface Workflow { jobs?: Job[]; } +export interface WorkflowUpdateRequest { + /** Whether the workflow is paused. */ + isPaused?: boolean; +} + export interface WorkflowConcurrency { /** * The maximum number of concurrent workflow runs. diff --git a/frontend/app/src/lib/hooks.ts b/frontend/app/src/lib/hooks.ts index 47a11633c..ed8f1c34e 100644 --- a/frontend/app/src/lib/hooks.ts +++ b/frontend/app/src/lib/hooks.ts @@ -1,4 +1,4 @@ -import { useToast } from '@/components/ui/use-toast'; +import { useToast } from '@/components/hooks/use-toast'; import { AxiosError } from 'axios'; import { Dispatch, SetStateAction } from 'react'; import api, { APIErrors } from './api'; diff --git a/frontend/app/src/pages/auth/hooks/use-error-param.ts b/frontend/app/src/pages/auth/hooks/use-error-param.ts index f31743922..af1ef73e4 100644 --- a/frontend/app/src/pages/auth/hooks/use-error-param.ts +++ b/frontend/app/src/pages/auth/hooks/use-error-param.ts @@ -1,4 +1,4 @@ -import { useToast } from '@/components/ui/use-toast'; +import { useToast } from '@/components/hooks/use-toast'; import { useEffect } from 'react'; import { useSearchParams } from 'react-router-dom'; diff --git a/frontend/app/src/pages/main/workflow-runs/$run/v2components/header.tsx b/frontend/app/src/pages/main/workflow-runs/$run/v2components/header.tsx index 8a68a8606..ac37b8018 100644 --- a/frontend/app/src/pages/main/workflow-runs/$run/v2components/header.tsx +++ b/frontend/app/src/pages/main/workflow-runs/$run/v2components/header.tsx @@ -25,7 +25,7 @@ import { } from '@/components/ui/breadcrumb'; import { formatDuration } from '@/lib/utils'; import RelativeDate from '@/components/molecules/relative-date'; -import { useToast } from '@/components/ui/use-toast'; +import { useToast } from '@/components/hooks/use-toast'; interface RunDetailHeaderProps { data?: WorkflowRunShape; @@ -57,7 +57,6 @@ const RunDetailHeader: React.FC = ({ toast({ title: 'Cancelling workflow run...', duration: 3000, - position: 'top-right', }); }, mutationFn: async () => { @@ -82,7 +81,6 @@ const RunDetailHeader: React.FC = ({ toast({ title: 'Replaying workflow run...', duration: 3000, - position: 'top-right', }); }, mutationFn: async () => { diff --git a/frontend/app/src/pages/main/workflows/$workflow/index.tsx b/frontend/app/src/pages/main/workflows/$workflow/index.tsx index 7a10e7317..7b3389280 100644 --- a/frontend/app/src/pages/main/workflows/$workflow/index.tsx +++ b/frontend/app/src/pages/main/workflows/$workflow/index.tsx @@ -1,5 +1,5 @@ import { Separator } from '@/components/ui/separator'; -import api, { queries } from '@/lib/api'; +import api, { queries, WorkflowUpdateRequest } from '@/lib/api'; import { useMutation, useQuery } from '@tanstack/react-query'; import { useNavigate, useOutletContext, useParams } from 'react-router-dom'; import invariant from 'tiny-invariant'; @@ -12,18 +12,25 @@ import { TenantContextType } from '@/lib/outlet'; import { TriggerWorkflowForm } from './components/trigger-workflow-form'; import { useState } from 'react'; import { Button } from '@/components/ui/button'; -import { useApiMetaIntegrations } from '@/lib/hooks'; +import { useApiError, useApiMetaIntegrations } from '@/lib/hooks'; import { Tabs, TabsContent, TabsList, TabsTrigger } from '@/components/ui/tabs'; import WorkflowGeneralSettings from './components/workflow-general-settings'; import { WorkflowRunsTable } from '../../workflow-runs/components/workflow-runs-table'; import { ConfirmDialog } from '@/components/molecules/confirm-dialog'; import { useTenantContext } from '@/lib/atoms'; +import { + DropdownMenu, + DropdownMenuContent, + DropdownMenuItem, + DropdownMenuTrigger, +} from '@/components/ui/dropdown-menu'; export default function ExpandedWorkflow() { const [tenant] = useTenantContext(); // TODO list previous versions and make selectable const [selectedVersion] = useState(); + const { handleApiError } = useApiError({}); invariant(tenant); @@ -45,6 +52,22 @@ export default function ExpandedWorkflow() { const navigate = useNavigate(); + const updateWorkflowMutation = useMutation({ + mutationKey: ['workflow:update', workflowQuery?.data?.metadata.id], + mutationFn: async (data: WorkflowUpdateRequest) => { + invariant(workflowQuery.data); + const res = await api.workflowUpdate(workflowQuery?.data?.metadata.id, { + ...data, + }); + + return res.data; + }, + onError: handleApiError, + onSuccess: () => { + workflowQuery.refetch(); + }, + }); + const deleteWorkflowMutation = useMutation({ mutationKey: ['workflow:delete', workflowQuery?.data?.metadata.id], mutationFn: async () => { @@ -86,11 +109,66 @@ export default function ExpandedWorkflow() { {currVersion} )} + + + {workflow.isPaused ? ( + { + updateWorkflowMutation.mutate({ isPaused: false }); + }} + > + Paused + + ) : ( + { + updateWorkflowMutation.mutate({ isPaused: true }); + }} + > + Active + + )} + + + + {workflow.isPaused ? ( +
{ + updateWorkflowMutation.mutate({ + isPaused: false, + }); + }} + > + Unpause runs +
+ ) : ( +
{ + updateWorkflowMutation.mutate({ + isPaused: true, + }); + }} + > + Pause runs +
+ )} +
+
+
- +
+ +
[] = [ + { + accessorKey: 'Status', + header: ({ column }) => ( + + ), + cell: ({ row }) => ( + <> + {row.original.isPaused ? ( + Paused + ) : ( + Active + )} + + ), + enableSorting: false, + enableHiding: false, + }, { accessorKey: 'name', header: ({ column }) => ( diff --git a/frontend/app/src/pages/main/workflows/components/workflow-table.tsx b/frontend/app/src/pages/main/workflows/components/workflow-table.tsx index 5d51190f7..59a585129 100644 --- a/frontend/app/src/pages/main/workflows/components/workflow-table.tsx +++ b/frontend/app/src/pages/main/workflows/components/workflow-table.tsx @@ -23,6 +23,7 @@ import { import { SortingState, VisibilityState } from '@tanstack/react-table'; import { BiCard, BiTable } from 'react-icons/bi'; import RelativeDate from '@/components/molecules/relative-date'; +import { Badge } from '@/components/ui/badge'; export function WorkflowTable() { const { tenant } = useOutletContext(); @@ -86,9 +87,16 @@ export function WorkflowTable() { className="border overflow-hidden shadow rounded-lg" >
-

- {data.name} -

+
+

+ {data.name} +

+ {data.isPaused ? ( + Paused + ) : ( + Active + )} +

{/* Last run{' '} {data.lastRunAt ? : 'never'} diff --git a/internal/services/controllers/retention/controller.go b/internal/services/controllers/retention/controller.go index 51beccaf4..680c4dea6 100644 --- a/internal/services/controllers/retention/controller.go +++ b/internal/services/controllers/retention/controller.go @@ -141,13 +141,16 @@ func New(fs ...RetentionControllerOpt) (*RetentionControllerImpl, error) { a.WithData(map[string]interface{}{"service": "retention-controller"}) return &RetentionControllerImpl{ - l: opts.l, - repo: opts.repo, - dv: opts.dv, - s: s, - tenantAlerter: opts.ta, - a: a, - p: opts.p, + l: opts.l, + repo: opts.repo, + dv: opts.dv, + s: s, + tenantAlerter: opts.ta, + a: a, + p: opts.p, + dataRetention: opts.dataRetention, + workerRetention: opts.workerRetention, + queueRetention: opts.queueRetention, }, nil } diff --git a/internal/services/controllers/workflows/controller.go b/internal/services/controllers/workflows/controller.go index 23781ae7e..1354bd09c 100644 --- a/internal/services/controllers/workflows/controller.go +++ b/internal/services/controllers/workflows/controller.go @@ -42,6 +42,7 @@ type WorkflowsControllerImpl struct { p *partition.Partition celParser *cel.CELParser processWorkflowEventsOps *queueutils.OperationPool + unpausedWorkflowRunsOps *queueutils.OperationPool bumpQueueOps *queueutils.OperationPool } @@ -158,6 +159,7 @@ func New(fs ...WorkflowsControllerOpt) (*WorkflowsControllerImpl, error) { } w.processWorkflowEventsOps = queueutils.NewOperationPool(w.l, time.Second*5, "process workflow events", w.processWorkflowEvents) + w.unpausedWorkflowRunsOps = queueutils.NewOperationPool(w.l, time.Second*5, "unpause workflow runs", w.unpauseWorkflowRuns) w.bumpQueueOps = queueutils.NewOperationPool(w.l, time.Second*5, "bump queue", w.runPollActiveQueuesTenant) return w, nil @@ -218,6 +220,18 @@ func (wc *WorkflowsControllerImpl) Start() (func() error, error) { return nil, fmt.Errorf("could not schedule process workflow run events: %w", err) } + _, err = wc.s.NewJob( + gocron.DurationJob(time.Second*1), + gocron.NewTask( + wc.runTenantUnpauseWorkflowRuns(ctx), + ), + ) + + if err != nil { + cancel() + return nil, fmt.Errorf("could not schedule unpause workflow runs: %w", err) + } + wc.s.Start() f := func(task *msgqueue.Message) error { @@ -661,6 +675,26 @@ func (wc *WorkflowsControllerImpl) runTenantProcessWorkflowRunEvents(ctx context } } +func (wc *WorkflowsControllerImpl) runTenantUnpauseWorkflowRuns(ctx context.Context) func() { + return func() { + wc.l.Debug().Msgf("partition: processing unpaused workflow runs") + + // list all tenants + tenants, err := wc.repo.Tenant().ListTenantsByControllerPartition(ctx, wc.p.GetControllerPartitionId()) + + if err != nil { + wc.l.Err(err).Msg("could not list tenants") + return + } + + for i := range tenants { + tenantId := sqlchelpers.UUIDToStr(tenants[i].ID) + + wc.unpausedWorkflowRunsOps.RunOrContinue(tenantId) + } + } +} + func (wc *WorkflowsControllerImpl) processWorkflowEvents(ctx context.Context, tenantId string) (bool, error) { ctx, span := telemetry.NewSpan(ctx, "process-workflow-events") defer span.End() @@ -676,3 +710,46 @@ func (wc *WorkflowsControllerImpl) processWorkflowEvents(ctx context.Context, te return res, nil } + +func (wc *WorkflowsControllerImpl) unpauseWorkflowRuns(ctx context.Context, tenantId string) (bool, error) { + ctx, span := telemetry.NewSpan(ctx, "unpause-workflow-runs") + defer span.End() + + dbCtx, cancel := context.WithTimeout(ctx, 300*time.Second) + defer cancel() + + toQueue, res, err := wc.repo.WorkflowRun().ProcessUnpausedWorkflowRuns(dbCtx, tenantId) + + if err != nil { + return false, fmt.Errorf("could not process unpaused workflow runs: %w", err) + } + + if toQueue != nil { + errGroup := new(errgroup.Group) + + for i := range toQueue { + row := toQueue[i] + + errGroup.Go(func() error { + workflowRunId := sqlchelpers.UUIDToStr(row.WorkflowRun.ID) + + wc.l.Info().Msgf("popped workflow run %s", workflowRunId) + workflowRun, err := wc.repo.WorkflowRun().GetWorkflowRunById(ctx, tenantId, workflowRunId) + + if err != nil { + return fmt.Errorf("could not get workflow run: %w", err) + } + + isPaused := workflowRun.IsPaused.Valid && workflowRun.IsPaused.Bool + + return wc.queueWorkflowRunJobs(ctx, workflowRun, isPaused) + }) + } + + if err := errGroup.Wait(); err != nil { + return false, fmt.Errorf("could not queue workflow runs: %w", err) + } + } + + return res, nil +} diff --git a/internal/services/controllers/workflows/queue.go b/internal/services/controllers/workflows/queue.go index c5d0af680..33338ad60 100644 --- a/internal/services/controllers/workflows/queue.go +++ b/internal/services/controllers/workflows/queue.go @@ -49,6 +49,8 @@ func (wc *WorkflowsControllerImpl) handleWorkflowRunQueued(ctx context.Context, return fmt.Errorf("could not get job run: %w", err) } + isPaused := workflowRun.IsPaused.Valid && workflowRun.IsPaused.Bool + workflowRunId := sqlchelpers.UUIDToStr(workflowRun.WorkflowRun.ID) servertel.WithWorkflowRunModel(span, workflowRun) @@ -108,7 +110,7 @@ func (wc *WorkflowsControllerImpl) handleWorkflowRunQueued(ctx context.Context, return fmt.Errorf("workflow run %s has concurrency settings but no group key run", workflowRunId) } - err = wc.queueWorkflowRunJobs(ctx, workflowRun) + err = wc.queueWorkflowRunJobs(ctx, workflowRun, isPaused) if err != nil { return fmt.Errorf("could not start workflow run: %w", err) @@ -308,12 +310,17 @@ func (wc *WorkflowsControllerImpl) scheduleGetGroupAction( return nil } -func (wc *WorkflowsControllerImpl) queueWorkflowRunJobs(ctx context.Context, workflowRun *dbsqlc.GetWorkflowRunRow) error { +func (wc *WorkflowsControllerImpl) queueWorkflowRunJobs(ctx context.Context, workflowRun *dbsqlc.GetWorkflowRunRow, isPaused bool) error { ctx, span := telemetry.NewSpan(ctx, "queue-workflow-run-jobs") // nolint:ineffassign defer span.End() tenantId := sqlchelpers.UUIDToStr(workflowRun.WorkflowRun.TenantId) workflowRunId := sqlchelpers.UUIDToStr(workflowRun.WorkflowRun.ID) + workflowId := sqlchelpers.UUIDToStr(workflowRun.WorkflowVersion.WorkflowId) + + if isPaused { + return wc.repo.WorkflowRun().QueuePausedWorkflowRun(ctx, tenantId, workflowId, workflowRunId) + } jobRuns, err := wc.repo.JobRun().ListJobRunsForWorkflowRun(ctx, tenantId, workflowRunId) @@ -627,7 +634,13 @@ func (wc *WorkflowsControllerImpl) queueByCancelInProgress(ctx context.Context, return fmt.Errorf("could not get workflow run: %w", err) } - return wc.queueWorkflowRunJobs(ctx, workflowRun) + isPaused := workflowRun.IsPaused.Valid && workflowRun.IsPaused.Bool + + if isPaused { + return nil + } + + return wc.queueWorkflowRunJobs(ctx, workflowRun, isPaused) }) } @@ -670,7 +683,13 @@ func (wc *WorkflowsControllerImpl) queueByGroupRoundRobin(ctx context.Context, t return fmt.Errorf("could not get workflow run: %w", err) } - return wc.queueWorkflowRunJobs(ctx, workflowRun) + isPaused := workflowRun.IsPaused.Valid && workflowRun.IsPaused.Bool + + if isPaused { + return nil + } + + return wc.queueWorkflowRunJobs(ctx, workflowRun, isPaused) }) } diff --git a/pkg/client/rest/gen.go b/pkg/client/rest/gen.go index 5b073e777..4523d7c0c 100644 --- a/pkg/client/rest/gen.go +++ b/pkg/client/rest/gen.go @@ -1118,6 +1118,9 @@ type Workflow struct { // Description The description of the workflow. Description *string `json:"description,omitempty"` + // IsPaused Whether the workflow is paused. + IsPaused *bool `json:"isPaused,omitempty"` + // Jobs The jobs of the workflow. Jobs *[]Job `json:"jobs,omitempty"` Metadata APIResourceMeta `json:"metadata"` @@ -1287,6 +1290,12 @@ type WorkflowTriggers struct { WorkflowVersionId *string `json:"workflow_version_id,omitempty"` } +// WorkflowUpdateRequest defines model for WorkflowUpdateRequest. +type WorkflowUpdateRequest struct { + // IsPaused Whether the workflow is paused. + IsPaused *bool `json:"isPaused,omitempty"` +} + // WorkflowVersion defines model for WorkflowVersion. type WorkflowVersion struct { Concurrency *WorkflowConcurrency `json:"concurrency,omitempty"` @@ -1590,6 +1599,9 @@ type UserCreateJSONRequestBody = UserRegisterRequest // WorkerUpdateJSONRequestBody defines body for WorkerUpdate for application/json ContentType. type WorkerUpdateJSONRequestBody = UpdateWorkerRequest +// WorkflowUpdateJSONRequestBody defines body for WorkflowUpdate for application/json ContentType. +type WorkflowUpdateJSONRequestBody = WorkflowUpdateRequest + // WorkflowRunCreateJSONRequestBody defines body for WorkflowRunCreate for application/json ContentType. type WorkflowRunCreateJSONRequestBody = TriggerWorkflowRunRequest @@ -1947,6 +1959,11 @@ type ClientInterface interface { // WorkflowGet request WorkflowGet(ctx context.Context, workflow openapi_types.UUID, reqEditors ...RequestEditorFn) (*http.Response, error) + // WorkflowUpdateWithBody request with any body + WorkflowUpdateWithBody(ctx context.Context, workflow openapi_types.UUID, contentType string, body io.Reader, reqEditors ...RequestEditorFn) (*http.Response, error) + + WorkflowUpdate(ctx context.Context, workflow openapi_types.UUID, body WorkflowUpdateJSONRequestBody, reqEditors ...RequestEditorFn) (*http.Response, error) + // WorkflowGetMetrics request WorkflowGetMetrics(ctx context.Context, workflow openapi_types.UUID, params *WorkflowGetMetricsParams, reqEditors ...RequestEditorFn) (*http.Response, error) @@ -3171,6 +3188,30 @@ func (c *Client) WorkflowGet(ctx context.Context, workflow openapi_types.UUID, r return c.Client.Do(req) } +func (c *Client) WorkflowUpdateWithBody(ctx context.Context, workflow openapi_types.UUID, contentType string, body io.Reader, reqEditors ...RequestEditorFn) (*http.Response, error) { + req, err := NewWorkflowUpdateRequestWithBody(c.Server, workflow, contentType, body) + if err != nil { + return nil, err + } + req = req.WithContext(ctx) + if err := c.applyEditors(ctx, req, reqEditors); err != nil { + return nil, err + } + return c.Client.Do(req) +} + +func (c *Client) WorkflowUpdate(ctx context.Context, workflow openapi_types.UUID, body WorkflowUpdateJSONRequestBody, reqEditors ...RequestEditorFn) (*http.Response, error) { + req, err := NewWorkflowUpdateRequest(c.Server, workflow, body) + if err != nil { + return nil, err + } + req = req.WithContext(ctx) + if err := c.applyEditors(ctx, req, reqEditors); err != nil { + return nil, err + } + return c.Client.Do(req) +} + func (c *Client) WorkflowGetMetrics(ctx context.Context, workflow openapi_types.UUID, params *WorkflowGetMetricsParams, reqEditors ...RequestEditorFn) (*http.Response, error) { req, err := NewWorkflowGetMetricsRequest(c.Server, workflow, params) if err != nil { @@ -6996,6 +7037,53 @@ func NewWorkflowGetRequest(server string, workflow openapi_types.UUID) (*http.Re return req, nil } +// NewWorkflowUpdateRequest calls the generic WorkflowUpdate builder with application/json body +func NewWorkflowUpdateRequest(server string, workflow openapi_types.UUID, body WorkflowUpdateJSONRequestBody) (*http.Request, error) { + var bodyReader io.Reader + buf, err := json.Marshal(body) + if err != nil { + return nil, err + } + bodyReader = bytes.NewReader(buf) + return NewWorkflowUpdateRequestWithBody(server, workflow, "application/json", bodyReader) +} + +// NewWorkflowUpdateRequestWithBody generates requests for WorkflowUpdate with any type of body +func NewWorkflowUpdateRequestWithBody(server string, workflow openapi_types.UUID, contentType string, body io.Reader) (*http.Request, error) { + var err error + + var pathParam0 string + + pathParam0, err = runtime.StyleParamWithLocation("simple", false, "workflow", runtime.ParamLocationPath, workflow) + if err != nil { + return nil, err + } + + serverURL, err := url.Parse(server) + if err != nil { + return nil, err + } + + operationPath := fmt.Sprintf("/api/v1/workflows/%s", pathParam0) + if operationPath[0] == '/' { + operationPath = "." + operationPath + } + + queryURL, err := serverURL.Parse(operationPath) + if err != nil { + return nil, err + } + + req, err := http.NewRequest("PATCH", queryURL.String(), body) + if err != nil { + return nil, err + } + + req.Header.Add("Content-Type", contentType) + + return req, nil +} + // NewWorkflowGetMetricsRequest generates requests for WorkflowGetMetrics func NewWorkflowGetMetricsRequest(server string, workflow openapi_types.UUID, params *WorkflowGetMetricsParams) (*http.Request, error) { var err error @@ -7517,6 +7605,11 @@ type ClientWithResponsesInterface interface { // WorkflowGetWithResponse request WorkflowGetWithResponse(ctx context.Context, workflow openapi_types.UUID, reqEditors ...RequestEditorFn) (*WorkflowGetResponse, error) + // WorkflowUpdateWithBodyWithResponse request with any body + WorkflowUpdateWithBodyWithResponse(ctx context.Context, workflow openapi_types.UUID, contentType string, body io.Reader, reqEditors ...RequestEditorFn) (*WorkflowUpdateResponse, error) + + WorkflowUpdateWithResponse(ctx context.Context, workflow openapi_types.UUID, body WorkflowUpdateJSONRequestBody, reqEditors ...RequestEditorFn) (*WorkflowUpdateResponse, error) + // WorkflowGetMetricsWithResponse request WorkflowGetMetricsWithResponse(ctx context.Context, workflow openapi_types.UUID, params *WorkflowGetMetricsParams, reqEditors ...RequestEditorFn) (*WorkflowGetMetricsResponse, error) @@ -9421,6 +9514,30 @@ func (r WorkflowGetResponse) StatusCode() int { return 0 } +type WorkflowUpdateResponse struct { + Body []byte + HTTPResponse *http.Response + JSON200 *Workflow + JSON400 *APIErrors + JSON403 *APIErrors +} + +// Status returns HTTPResponse.Status +func (r WorkflowUpdateResponse) Status() string { + if r.HTTPResponse != nil { + return r.HTTPResponse.Status + } + return http.StatusText(0) +} + +// StatusCode returns HTTPResponse.StatusCode +func (r WorkflowUpdateResponse) StatusCode() int { + if r.HTTPResponse != nil { + return r.HTTPResponse.StatusCode + } + return 0 +} + type WorkflowGetMetricsResponse struct { Body []byte HTTPResponse *http.Response @@ -10384,6 +10501,23 @@ func (c *ClientWithResponses) WorkflowGetWithResponse(ctx context.Context, workf return ParseWorkflowGetResponse(rsp) } +// WorkflowUpdateWithBodyWithResponse request with arbitrary body returning *WorkflowUpdateResponse +func (c *ClientWithResponses) WorkflowUpdateWithBodyWithResponse(ctx context.Context, workflow openapi_types.UUID, contentType string, body io.Reader, reqEditors ...RequestEditorFn) (*WorkflowUpdateResponse, error) { + rsp, err := c.WorkflowUpdateWithBody(ctx, workflow, contentType, body, reqEditors...) + if err != nil { + return nil, err + } + return ParseWorkflowUpdateResponse(rsp) +} + +func (c *ClientWithResponses) WorkflowUpdateWithResponse(ctx context.Context, workflow openapi_types.UUID, body WorkflowUpdateJSONRequestBody, reqEditors ...RequestEditorFn) (*WorkflowUpdateResponse, error) { + rsp, err := c.WorkflowUpdate(ctx, workflow, body, reqEditors...) + if err != nil { + return nil, err + } + return ParseWorkflowUpdateResponse(rsp) +} + // WorkflowGetMetricsWithResponse request returning *WorkflowGetMetricsResponse func (c *ClientWithResponses) WorkflowGetMetricsWithResponse(ctx context.Context, workflow openapi_types.UUID, params *WorkflowGetMetricsParams, reqEditors ...RequestEditorFn) (*WorkflowGetMetricsResponse, error) { rsp, err := c.WorkflowGetMetrics(ctx, workflow, params, reqEditors...) @@ -13503,6 +13637,46 @@ func ParseWorkflowGetResponse(rsp *http.Response) (*WorkflowGetResponse, error) return response, nil } +// ParseWorkflowUpdateResponse parses an HTTP response from a WorkflowUpdateWithResponse call +func ParseWorkflowUpdateResponse(rsp *http.Response) (*WorkflowUpdateResponse, error) { + bodyBytes, err := io.ReadAll(rsp.Body) + defer func() { _ = rsp.Body.Close() }() + if err != nil { + return nil, err + } + + response := &WorkflowUpdateResponse{ + Body: bodyBytes, + HTTPResponse: rsp, + } + + switch { + case strings.Contains(rsp.Header.Get("Content-Type"), "json") && rsp.StatusCode == 200: + var dest Workflow + if err := json.Unmarshal(bodyBytes, &dest); err != nil { + return nil, err + } + response.JSON200 = &dest + + case strings.Contains(rsp.Header.Get("Content-Type"), "json") && rsp.StatusCode == 400: + var dest APIErrors + if err := json.Unmarshal(bodyBytes, &dest); err != nil { + return nil, err + } + response.JSON400 = &dest + + case strings.Contains(rsp.Header.Get("Content-Type"), "json") && rsp.StatusCode == 403: + var dest APIErrors + if err := json.Unmarshal(bodyBytes, &dest); err != nil { + return nil, err + } + response.JSON403 = &dest + + } + + return response, nil +} + // ParseWorkflowGetMetricsResponse parses an HTTP response from a WorkflowGetMetricsWithResponse call func ParseWorkflowGetMetricsResponse(rsp *http.Response) (*WorkflowGetMetricsResponse, error) { bodyBytes, err := io.ReadAll(rsp.Body) diff --git a/pkg/repository/prisma/db/db_gen.go b/pkg/repository/prisma/db/db_gen.go index 82a1eb463..d9000e9e8 100644 --- a/pkg/repository/prisma/db/db_gen.go +++ b/pkg/repository/prisma/db/db_gen.go @@ -537,6 +537,8 @@ model Workflow { // the parent tenant tenantId String @db.Uuid + isPaused Boolean? @default(false) + // the workflow name name String @@ -1421,6 +1423,7 @@ enum InternalQueue { WORKER_SEMAPHORE_COUNT STEP_RUN_UPDATE WORKFLOW_RUN_UPDATE + WORKFLOW_RUN_PAUSED } model InternalQueueItem { @@ -2272,6 +2275,7 @@ const ( InternalQueueWorkerSemaphoreCount InternalQueue = "WORKER_SEMAPHORE_COUNT" InternalQueueStepRunUpdate InternalQueue = "STEP_RUN_UPDATE" InternalQueueWorkflowRunUpdate InternalQueue = "WORKFLOW_RUN_UPDATE" + InternalQueueWorkflowRunPaused InternalQueue = "WORKFLOW_RUN_PAUSED" ) type RawInternalQueue InternalQueue @@ -2587,6 +2591,7 @@ const ( WorkflowScalarFieldEnumUpdatedAt WorkflowScalarFieldEnum = "updatedAt" WorkflowScalarFieldEnumDeletedAt WorkflowScalarFieldEnum = "deletedAt" WorkflowScalarFieldEnumTenantID WorkflowScalarFieldEnum = "tenantId" + WorkflowScalarFieldEnumIsPaused WorkflowScalarFieldEnum = "isPaused" WorkflowScalarFieldEnumName WorkflowScalarFieldEnum = "name" WorkflowScalarFieldEnumDescription WorkflowScalarFieldEnum = "description" ) @@ -3638,6 +3643,8 @@ const workflowFieldDeletedAt workflowPrismaFields = "deletedAt" const workflowFieldTenantID workflowPrismaFields = "tenantId" +const workflowFieldIsPaused workflowPrismaFields = "isPaused" + const workflowFieldName workflowPrismaFields = "name" const workflowFieldDescription workflowPrismaFields = "description" @@ -8768,6 +8775,7 @@ type InnerWorkflow struct { UpdatedAt DateTime `json:"updatedAt"` DeletedAt *DateTime `json:"deletedAt,omitempty"` TenantID string `json:"tenantId"` + IsPaused *bool `json:"isPaused,omitempty"` Name string `json:"name"` Description *string `json:"description,omitempty"` } @@ -8779,6 +8787,7 @@ type RawWorkflowModel struct { UpdatedAt RawDateTime `json:"updatedAt"` DeletedAt *RawDateTime `json:"deletedAt,omitempty"` TenantID RawString `json:"tenantId"` + IsPaused *RawBoolean `json:"isPaused,omitempty"` Name RawString `json:"name"` Description *RawString `json:"description,omitempty"` } @@ -8797,6 +8806,13 @@ func (r WorkflowModel) DeletedAt() (value DateTime, ok bool) { return *r.InnerWorkflow.DeletedAt, true } +func (r WorkflowModel) IsPaused() (value Boolean, ok bool) { + if r.InnerWorkflow.IsPaused == nil { + return value, false + } + return *r.InnerWorkflow.IsPaused, true +} + func (r WorkflowModel) Description() (value String, ok bool) { if r.InnerWorkflow.Description == nil { return value, false @@ -61789,6 +61805,11 @@ type workflowQuery struct { // @required TenantID workflowQueryTenantIDString + // IsPaused + // + // @optional + IsPaused workflowQueryIsPausedBoolean + // Name // // @required @@ -63547,6 +63568,119 @@ func (r workflowQueryTenantIDString) Field() workflowPrismaFields { return workflowFieldTenantID } +// base struct +type workflowQueryIsPausedBoolean struct{} + +// Set the optional value of IsPaused +func (r workflowQueryIsPausedBoolean) Set(value bool) workflowSetParam { + + return workflowSetParam{ + data: builder.Field{ + Name: "isPaused", + Value: value, + }, + } + +} + +// Set the optional value of IsPaused dynamically +func (r workflowQueryIsPausedBoolean) SetIfPresent(value *Boolean) workflowSetParam { + if value == nil { + return workflowSetParam{} + } + + return r.Set(*value) +} + +// Set the optional value of IsPaused dynamically +func (r workflowQueryIsPausedBoolean) SetOptional(value *Boolean) workflowSetParam { + if value == nil { + + var v *bool + return workflowSetParam{ + data: builder.Field{ + Name: "isPaused", + Value: v, + }, + } + } + + return r.Set(*value) +} + +func (r workflowQueryIsPausedBoolean) Equals(value bool) workflowWithPrismaIsPausedEqualsParam { + + return workflowWithPrismaIsPausedEqualsParam{ + data: builder.Field{ + Name: "isPaused", + Fields: []builder.Field{ + { + Name: "equals", + Value: value, + }, + }, + }, + } +} + +func (r workflowQueryIsPausedBoolean) EqualsIfPresent(value *bool) workflowWithPrismaIsPausedEqualsParam { + if value == nil { + return workflowWithPrismaIsPausedEqualsParam{} + } + return r.Equals(*value) +} + +func (r workflowQueryIsPausedBoolean) EqualsOptional(value *Boolean) workflowDefaultParam { + return workflowDefaultParam{ + data: builder.Field{ + Name: "isPaused", + Fields: []builder.Field{ + { + Name: "equals", + Value: value, + }, + }, + }, + } +} + +func (r workflowQueryIsPausedBoolean) IsNull() workflowDefaultParam { + var str *string = nil + return workflowDefaultParam{ + data: builder.Field{ + Name: "isPaused", + Fields: []builder.Field{ + { + Name: "equals", + Value: str, + }, + }, + }, + } +} + +func (r workflowQueryIsPausedBoolean) Order(direction SortOrder) workflowDefaultParam { + return workflowDefaultParam{ + data: builder.Field{ + Name: "isPaused", + Value: direction, + }, + } +} + +func (r workflowQueryIsPausedBoolean) Cursor(cursor bool) workflowCursorParam { + return workflowCursorParam{ + data: builder.Field{ + Name: "isPaused", + Value: cursor, + }, + } +} + +func (r workflowQueryIsPausedBoolean) Field() workflowPrismaFields { + return workflowFieldIsPaused +} + // base struct type workflowQueryNameString struct{} @@ -216415,6 +216549,7 @@ var workflowOutput = []builder.Output{ {Name: "updatedAt"}, {Name: "deletedAt"}, {Name: "tenantId"}, + {Name: "isPaused"}, {Name: "name"}, {Name: "description"}, } @@ -216973,6 +217108,84 @@ func (p workflowWithPrismaTenantIDEqualsUniqueParam) tenantIDField() {} func (workflowWithPrismaTenantIDEqualsUniqueParam) unique() {} func (workflowWithPrismaTenantIDEqualsUniqueParam) equals() {} +type WorkflowWithPrismaIsPausedEqualsSetParam interface { + field() builder.Field + getQuery() builder.Query + equals() + workflowModel() + isPausedField() +} + +type WorkflowWithPrismaIsPausedSetParam interface { + field() builder.Field + getQuery() builder.Query + workflowModel() + isPausedField() +} + +type workflowWithPrismaIsPausedSetParam struct { + data builder.Field + query builder.Query +} + +func (p workflowWithPrismaIsPausedSetParam) field() builder.Field { + return p.data +} + +func (p workflowWithPrismaIsPausedSetParam) getQuery() builder.Query { + return p.query +} + +func (p workflowWithPrismaIsPausedSetParam) workflowModel() {} + +func (p workflowWithPrismaIsPausedSetParam) isPausedField() {} + +type WorkflowWithPrismaIsPausedWhereParam interface { + field() builder.Field + getQuery() builder.Query + workflowModel() + isPausedField() +} + +type workflowWithPrismaIsPausedEqualsParam struct { + data builder.Field + query builder.Query +} + +func (p workflowWithPrismaIsPausedEqualsParam) field() builder.Field { + return p.data +} + +func (p workflowWithPrismaIsPausedEqualsParam) getQuery() builder.Query { + return p.query +} + +func (p workflowWithPrismaIsPausedEqualsParam) workflowModel() {} + +func (p workflowWithPrismaIsPausedEqualsParam) isPausedField() {} + +func (workflowWithPrismaIsPausedSetParam) settable() {} +func (workflowWithPrismaIsPausedEqualsParam) equals() {} + +type workflowWithPrismaIsPausedEqualsUniqueParam struct { + data builder.Field + query builder.Query +} + +func (p workflowWithPrismaIsPausedEqualsUniqueParam) field() builder.Field { + return p.data +} + +func (p workflowWithPrismaIsPausedEqualsUniqueParam) getQuery() builder.Query { + return p.query +} + +func (p workflowWithPrismaIsPausedEqualsUniqueParam) workflowModel() {} +func (p workflowWithPrismaIsPausedEqualsUniqueParam) isPausedField() {} + +func (workflowWithPrismaIsPausedEqualsUniqueParam) unique() {} +func (workflowWithPrismaIsPausedEqualsUniqueParam) equals() {} + type WorkflowWithPrismaNameEqualsSetParam interface { field() builder.Field getQuery() builder.Query diff --git a/pkg/repository/prisma/dbsqlc/models.go b/pkg/repository/prisma/dbsqlc/models.go index b66e606a8..95a218946 100644 --- a/pkg/repository/prisma/dbsqlc/models.go +++ b/pkg/repository/prisma/dbsqlc/models.go @@ -61,6 +61,7 @@ const ( InternalQueueWORKERSEMAPHORECOUNT InternalQueue = "WORKER_SEMAPHORE_COUNT" InternalQueueSTEPRUNUPDATE InternalQueue = "STEP_RUN_UPDATE" InternalQueueWORKFLOWRUNUPDATE InternalQueue = "WORKFLOW_RUN_UPDATE" + InternalQueueWORKFLOWRUNPAUSED InternalQueue = "WORKFLOW_RUN_PAUSED" ) func (e *InternalQueue) Scan(src interface{}) error { @@ -1511,6 +1512,7 @@ type Workflow struct { TenantId pgtype.UUID `json:"tenantId"` Name string `json:"name"` Description pgtype.Text `json:"description"` + IsPaused pgtype.Bool `json:"isPaused"` } type WorkflowConcurrency struct { diff --git a/pkg/repository/prisma/dbsqlc/schema.sql b/pkg/repository/prisma/dbsqlc/schema.sql index 9e994f089..e9269ea13 100644 --- a/pkg/repository/prisma/dbsqlc/schema.sql +++ b/pkg/repository/prisma/dbsqlc/schema.sql @@ -2,7 +2,7 @@ CREATE TYPE "ConcurrencyLimitStrategy" AS ENUM ('CANCEL_IN_PROGRESS', 'DROP_NEWEST', 'QUEUE_NEWEST', 'GROUP_ROUND_ROBIN'); -- CreateEnum -CREATE TYPE "InternalQueue" AS ENUM ('WORKER_SEMAPHORE_COUNT', 'STEP_RUN_UPDATE', 'WORKFLOW_RUN_UPDATE'); +CREATE TYPE "InternalQueue" AS ENUM ('WORKER_SEMAPHORE_COUNT', 'STEP_RUN_UPDATE', 'WORKFLOW_RUN_UPDATE', 'WORKFLOW_RUN_PAUSED'); -- CreateEnum CREATE TYPE "InviteLinkStatus" AS ENUM ('PENDING', 'ACCEPTED', 'REJECTED'); @@ -752,6 +752,7 @@ CREATE TABLE "Workflow" ( "tenantId" UUID NOT NULL, "name" TEXT NOT NULL, "description" TEXT, + "isPaused" BOOLEAN DEFAULT false, CONSTRAINT "Workflow_pkey" PRIMARY KEY ("id") ); diff --git a/pkg/repository/prisma/dbsqlc/workflow_runs.sql b/pkg/repository/prisma/dbsqlc/workflow_runs.sql index d88605ab9..04b81431a 100644 --- a/pkg/repository/prisma/dbsqlc/workflow_runs.sql +++ b/pkg/repository/prisma/dbsqlc/workflow_runs.sql @@ -779,6 +779,7 @@ SELECT -- waiting on https://github.com/sqlc-dev/sqlc/pull/2858 for nullable fields wc."limitStrategy" as "concurrencyLimitStrategy", wc."maxRuns" as "concurrencyMaxRuns", + workflow."isPaused" as "isPaused", wc."concurrencyGroupExpression" as "concurrencyGroupExpression", groupKeyRun."id" as "getGroupKeyRunId" FROM diff --git a/pkg/repository/prisma/dbsqlc/workflow_runs.sql.go b/pkg/repository/prisma/dbsqlc/workflow_runs.sql.go index 1447c1793..73e92b55d 100644 --- a/pkg/repository/prisma/dbsqlc/workflow_runs.sql.go +++ b/pkg/repository/prisma/dbsqlc/workflow_runs.sql.go @@ -986,6 +986,7 @@ SELECT -- waiting on https://github.com/sqlc-dev/sqlc/pull/2858 for nullable fields wc."limitStrategy" as "concurrencyLimitStrategy", wc."maxRuns" as "concurrencyMaxRuns", + workflow."isPaused" as "isPaused", wc."concurrencyGroupExpression" as "concurrencyGroupExpression", groupKeyRun."id" as "getGroupKeyRunId" FROM @@ -1020,6 +1021,7 @@ type GetWorkflowRunRow struct { WorkflowName pgtype.Text `json:"workflowName"` ConcurrencyLimitStrategy NullConcurrencyLimitStrategy `json:"concurrencyLimitStrategy"` ConcurrencyMaxRuns pgtype.Int4 `json:"concurrencyMaxRuns"` + IsPaused pgtype.Bool `json:"isPaused"` ConcurrencyGroupExpression pgtype.Text `json:"concurrencyGroupExpression"` GetGroupKeyRunId pgtype.UUID `json:"getGroupKeyRunId"` } @@ -1080,6 +1082,7 @@ func (q *Queries) GetWorkflowRun(ctx context.Context, db DBTX, arg GetWorkflowRu &i.WorkflowName, &i.ConcurrencyLimitStrategy, &i.ConcurrencyMaxRuns, + &i.IsPaused, &i.ConcurrencyGroupExpression, &i.GetGroupKeyRunId, ); err != nil { @@ -1125,7 +1128,7 @@ const getWorkflowRunById = `-- name: GetWorkflowRunById :one SELECT r."createdAt", r."updatedAt", r."deletedAt", r."tenantId", r."workflowVersionId", r.status, r.error, r."startedAt", r."finishedAt", r."concurrencyGroupId", r."displayName", r.id, r."childIndex", r."childKey", r."parentId", r."parentStepRunId", r."additionalMetadata", r.duration, r.priority, wv.id, wv."createdAt", wv."updatedAt", wv."deletedAt", wv.version, wv."order", wv."workflowId", wv.checksum, wv."scheduleTimeout", wv."onFailureJobId", wv.sticky, wv.kind, wv."defaultPriority", - w.id, w."createdAt", w."updatedAt", w."deletedAt", w."tenantId", w.name, w.description, + w.id, w."createdAt", w."updatedAt", w."deletedAt", w."tenantId", w.name, w.description, w."isPaused", tb.id, tb."createdAt", tb."updatedAt", tb."deletedAt", tb."tenantId", tb."eventId", tb."cronParentId", tb."cronSchedule", tb."scheduledId", tb.input, tb."parentId" FROM "WorkflowRun" r @@ -1214,6 +1217,7 @@ func (q *Queries) GetWorkflowRunById(ctx context.Context, db DBTX, arg GetWorkfl &i.Workflow.TenantId, &i.Workflow.Name, &i.Workflow.Description, + &i.Workflow.IsPaused, &i.WorkflowRunTriggeredBy.ID, &i.WorkflowRunTriggeredBy.CreatedAt, &i.WorkflowRunTriggeredBy.UpdatedAt, @@ -1514,7 +1518,7 @@ func (q *Queries) ListWorkflowRunEventsByWorkflowRunId(ctx context.Context, db D const listWorkflowRuns = `-- name: ListWorkflowRuns :many SELECT runs."createdAt", runs."updatedAt", runs."deletedAt", runs."tenantId", runs."workflowVersionId", runs.status, runs.error, runs."startedAt", runs."finishedAt", runs."concurrencyGroupId", runs."displayName", runs.id, runs."childIndex", runs."childKey", runs."parentId", runs."parentStepRunId", runs."additionalMetadata", runs.duration, runs.priority, - workflow.id, workflow."createdAt", workflow."updatedAt", workflow."deletedAt", workflow."tenantId", workflow.name, workflow.description, + workflow.id, workflow."createdAt", workflow."updatedAt", workflow."deletedAt", workflow."tenantId", workflow.name, workflow.description, workflow."isPaused", runtriggers.id, runtriggers."createdAt", runtriggers."updatedAt", runtriggers."deletedAt", runtriggers."tenantId", runtriggers."eventId", runtriggers."cronParentId", runtriggers."cronSchedule", runtriggers."scheduledId", runtriggers.input, runtriggers."parentId", workflowversion.id, workflowversion."createdAt", workflowversion."updatedAt", workflowversion."deletedAt", workflowversion.version, workflowversion."order", workflowversion."workflowId", workflowversion.checksum, workflowversion."scheduleTimeout", workflowversion."onFailureJobId", workflowversion.sticky, workflowversion.kind, workflowversion."defaultPriority", -- waiting on https://github.com/sqlc-dev/sqlc/pull/2858 for nullable events field @@ -1701,6 +1705,7 @@ func (q *Queries) ListWorkflowRuns(ctx context.Context, db DBTX, arg ListWorkflo &i.Workflow.TenantId, &i.Workflow.Name, &i.Workflow.Description, + &i.Workflow.IsPaused, &i.WorkflowRunTriggeredBy.ID, &i.WorkflowRunTriggeredBy.CreatedAt, &i.WorkflowRunTriggeredBy.UpdatedAt, diff --git a/pkg/repository/prisma/dbsqlc/workflows.sql b/pkg/repository/prisma/dbsqlc/workflows.sql index d81ddcff2..e09e8db9f 100644 --- a/pkg/repository/prisma/dbsqlc/workflows.sql +++ b/pkg/repository/prisma/dbsqlc/workflows.sql @@ -486,6 +486,49 @@ SET WHERE "id" = @id::uuid RETURNING *; +-- name: ListPausedWorkflows :many +SELECT + "id" +FROM + "Workflow" +WHERE + "tenantId" = @tenantId::uuid AND + "isPaused" = true AND + "deletedAt" IS NULL; + +-- name: UpdateWorkflow :one +UPDATE "Workflow" +SET + "updatedAt" = CURRENT_TIMESTAMP, + "isPaused" = coalesce(sqlc.narg('isPaused')::boolean, "isPaused") +WHERE "id" = @id::uuid +RETURNING *; + +-- name: HandleWorkflowUnpaused :exec +WITH matching_qis AS ( + -- We know that we're going to need to scan all the queue items in this queue + -- for the tenant, so we write this query in such a way that the index is used. + SELECT + qi."id" + FROM + "InternalQueueItem" qi + WHERE + qi."isQueued" = true + AND qi."tenantId" = @tenantId::uuid + AND qi."queue" = 'WORKFLOW_RUN_PAUSED' + AND qi."priority" = 1 + ORDER BY + qi."id" DESC +) +UPDATE "InternalQueueItem" +-- We update all the queue items to have a higher priority so we can unpause them +SET "priority" = 4 +FROM + matching_qis +WHERE + "InternalQueueItem"."id" = matching_qis."id" + AND "data"->>'workflow_id' = @workflowId::text; + -- name: GetWorkflowWorkerCount :one WITH UniqueWorkers AS ( SELECT DISTINCT w."id" AS workerId diff --git a/pkg/repository/prisma/dbsqlc/workflows.sql.go b/pkg/repository/prisma/dbsqlc/workflows.sql.go index c0814c71a..5a2b87845 100644 --- a/pkg/repository/prisma/dbsqlc/workflows.sql.go +++ b/pkg/repository/prisma/dbsqlc/workflows.sql.go @@ -458,7 +458,7 @@ INSERT INTO "Workflow" ( $5::uuid, $6::text, $7::text -) RETURNING id, "createdAt", "updatedAt", "deletedAt", "tenantId", name, description +) RETURNING id, "createdAt", "updatedAt", "deletedAt", "tenantId", name, description, "isPaused" ` type CreateWorkflowParams struct { @@ -490,6 +490,7 @@ func (q *Queries) CreateWorkflow(ctx context.Context, db DBTX, arg CreateWorkflo &i.TenantId, &i.Name, &i.Description, + &i.IsPaused, ) return &i, err } @@ -756,7 +757,7 @@ func (q *Queries) CreateWorkflowVersion(ctx context.Context, db DBTX, arg Create const getWorkflowById = `-- name: GetWorkflowById :one SELECT - w.id, w."createdAt", w."updatedAt", w."deletedAt", w."tenantId", w.name, w.description, + w.id, w."createdAt", w."updatedAt", w."deletedAt", w."tenantId", w.name, w.description, w."isPaused", wv."id" as "workflowVersionId" FROM "Workflow" as w @@ -785,6 +786,7 @@ func (q *Queries) GetWorkflowById(ctx context.Context, db DBTX, id pgtype.UUID) &i.Workflow.TenantId, &i.Workflow.Name, &i.Workflow.Description, + &i.Workflow.IsPaused, &i.WorkflowVersionId, ) return &i, err @@ -792,7 +794,7 @@ func (q *Queries) GetWorkflowById(ctx context.Context, db DBTX, id pgtype.UUID) const getWorkflowByName = `-- name: GetWorkflowByName :one SELECT - id, "createdAt", "updatedAt", "deletedAt", "tenantId", name, description + id, "createdAt", "updatedAt", "deletedAt", "tenantId", name, description, "isPaused" FROM "Workflow" as workflows WHERE @@ -817,6 +819,7 @@ func (q *Queries) GetWorkflowByName(ctx context.Context, db DBTX, arg GetWorkflo &i.TenantId, &i.Name, &i.Description, + &i.IsPaused, ) return &i, err } @@ -844,7 +847,7 @@ func (q *Queries) GetWorkflowLatestVersion(ctx context.Context, db DBTX, workflo const getWorkflowVersionById = `-- name: GetWorkflowVersionById :one SELECT wv.id, wv."createdAt", wv."updatedAt", wv."deletedAt", wv.version, wv."order", wv."workflowId", wv.checksum, wv."scheduleTimeout", wv."onFailureJobId", wv.sticky, wv.kind, wv."defaultPriority", - w.id, w."createdAt", w."updatedAt", w."deletedAt", w."tenantId", w.name, w.description, + w.id, w."createdAt", w."updatedAt", w."deletedAt", w."tenantId", w.name, w.description, w."isPaused", wc."id" as "concurrencyId", wc."maxRuns" as "concurrencyMaxRuns", wc."getConcurrencyGroupId" as "concurrencyGroupId", @@ -892,6 +895,7 @@ func (q *Queries) GetWorkflowVersionById(ctx context.Context, db DBTX, id pgtype &i.Workflow.TenantId, &i.Workflow.Name, &i.Workflow.Description, + &i.Workflow.IsPaused, &i.ConcurrencyId, &i.ConcurrencyMaxRuns, &i.ConcurrencyGroupId, @@ -1132,6 +1136,42 @@ func (q *Queries) GetWorkflowWorkerCount(ctx context.Context, db DBTX, arg GetWo return &i, err } +const handleWorkflowUnpaused = `-- name: HandleWorkflowUnpaused :exec +WITH matching_qis AS ( + -- We know that we're going to need to scan all the queue items in this queue + -- for the tenant, so we write this query in such a way that the index is used. + SELECT + qi."id" + FROM + "InternalQueueItem" qi + WHERE + qi."isQueued" = true + AND qi."tenantId" = $2::uuid + AND qi."queue" = 'WORKFLOW_RUN_PAUSED' + AND qi."priority" = 1 + ORDER BY + qi."id" DESC +) +UPDATE "InternalQueueItem" +SET "priority" = 4 +FROM + matching_qis +WHERE + "InternalQueueItem"."id" = matching_qis."id" + AND "data"->>'workflow_id' = $1::text +` + +type HandleWorkflowUnpausedParams struct { + Workflowid string `json:"workflowid"` + Tenantid pgtype.UUID `json:"tenantid"` +} + +// We update all the queue items to have a higher priority so we can unpause them +func (q *Queries) HandleWorkflowUnpaused(ctx context.Context, db DBTX, arg HandleWorkflowUnpausedParams) error { + _, err := db.Exec(ctx, handleWorkflowUnpaused, arg.Workflowid, arg.Tenantid) + return err +} + const linkOnFailureJob = `-- name: LinkOnFailureJob :one UPDATE "WorkflowVersion" SET "onFailureJobId" = $1::uuid @@ -1165,9 +1205,40 @@ func (q *Queries) LinkOnFailureJob(ctx context.Context, db DBTX, arg LinkOnFailu return &i, err } +const listPausedWorkflows = `-- name: ListPausedWorkflows :many +SELECT + "id" +FROM + "Workflow" +WHERE + "tenantId" = $1::uuid AND + "isPaused" = true AND + "deletedAt" IS NULL +` + +func (q *Queries) ListPausedWorkflows(ctx context.Context, db DBTX, tenantid pgtype.UUID) ([]pgtype.UUID, error) { + rows, err := db.Query(ctx, listPausedWorkflows, tenantid) + if err != nil { + return nil, err + } + defer rows.Close() + var items []pgtype.UUID + for rows.Next() { + var id pgtype.UUID + if err := rows.Scan(&id); err != nil { + return nil, err + } + items = append(items, id) + } + if err := rows.Err(); err != nil { + return nil, err + } + return items, nil +} + const listWorkflows = `-- name: ListWorkflows :many SELECT - workflows.id, workflows."createdAt", workflows."updatedAt", workflows."deletedAt", workflows."tenantId", workflows.name, workflows.description + workflows.id, workflows."createdAt", workflows."updatedAt", workflows."deletedAt", workflows."tenantId", workflows.name, workflows.description, workflows."isPaused" FROM "Workflow" as workflows WHERE @@ -1215,6 +1286,7 @@ func (q *Queries) ListWorkflows(ctx context.Context, db DBTX, arg ListWorkflowsP &i.Workflow.TenantId, &i.Workflow.Name, &i.Workflow.Description, + &i.Workflow.IsPaused, ); err != nil { return nil, err } @@ -1384,7 +1456,7 @@ SET "name" = "name" || '-' || gen_random_uuid(), "deletedAt" = CURRENT_TIMESTAMP WHERE "id" = $1::uuid -RETURNING id, "createdAt", "updatedAt", "deletedAt", "tenantId", name, description +RETURNING id, "createdAt", "updatedAt", "deletedAt", "tenantId", name, description, "isPaused" ` func (q *Queries) SoftDeleteWorkflow(ctx context.Context, db DBTX, id pgtype.UUID) (*Workflow, error) { @@ -1398,6 +1470,37 @@ func (q *Queries) SoftDeleteWorkflow(ctx context.Context, db DBTX, id pgtype.UUI &i.TenantId, &i.Name, &i.Description, + &i.IsPaused, + ) + return &i, err +} + +const updateWorkflow = `-- name: UpdateWorkflow :one +UPDATE "Workflow" +SET + "updatedAt" = CURRENT_TIMESTAMP, + "isPaused" = coalesce($1::boolean, "isPaused") +WHERE "id" = $2::uuid +RETURNING id, "createdAt", "updatedAt", "deletedAt", "tenantId", name, description, "isPaused" +` + +type UpdateWorkflowParams struct { + IsPaused pgtype.Bool `json:"isPaused"` + ID pgtype.UUID `json:"id"` +} + +func (q *Queries) UpdateWorkflow(ctx context.Context, db DBTX, arg UpdateWorkflowParams) (*Workflow, error) { + row := db.QueryRow(ctx, updateWorkflow, arg.IsPaused, arg.ID) + var i Workflow + err := row.Scan( + &i.ID, + &i.CreatedAt, + &i.UpdatedAt, + &i.DeletedAt, + &i.TenantId, + &i.Name, + &i.Description, + &i.IsPaused, ) return &i, err } diff --git a/pkg/repository/prisma/workflow.go b/pkg/repository/prisma/workflow.go index 39752fab9..cae82ce9a 100644 --- a/pkg/repository/prisma/workflow.go +++ b/pkg/repository/prisma/workflow.go @@ -115,6 +115,57 @@ func (r *workflowAPIRepository) ListWorkflows(tenantId string, opts *repository. return res, nil } +func (r *workflowAPIRepository) UpdateWorkflow(ctx context.Context, tenantId, workflowId string, opts *repository.UpdateWorkflowOpts) (*dbsqlc.Workflow, error) { + if err := r.v.Validate(opts); err != nil { + return nil, err + } + + pgWorkflowId := sqlchelpers.UUIDFromStr(workflowId) + + params := dbsqlc.UpdateWorkflowParams{ + ID: pgWorkflowId, + } + + if opts.IsPaused != nil { + params.IsPaused = pgtype.Bool{ + Valid: true, + Bool: *opts.IsPaused, + } + } + + tx, commit, rollback, err := prepareTx(ctx, r.pool, r.l, 25000) + + if err != nil { + return nil, err + } + + defer rollback() + + workflow, err := r.queries.UpdateWorkflow(ctx, tx, params) + + if err != nil { + return nil, err + } + + // if we're setting to an unpaused state, update internal queue items + if opts.IsPaused != nil && !*opts.IsPaused { + err = r.queries.HandleWorkflowUnpaused(ctx, tx, dbsqlc.HandleWorkflowUnpausedParams{ + Workflowid: workflowId, + Tenantid: sqlchelpers.UUIDFromStr(tenantId), + }) + + if err != nil { + return nil, err + } + } + + if err := commit(ctx); err != nil { + return nil, err + } + + return workflow, nil +} + func (r *workflowAPIRepository) GetWorkflowById(ctx context.Context, workflowId string) (*dbsqlc.GetWorkflowByIdRow, error) { return r.queries.GetWorkflowById(context.Background(), r.pool, sqlchelpers.UUIDFromStr(workflowId)) diff --git a/pkg/repository/prisma/workflow_run.go b/pkg/repository/prisma/workflow_run.go index fbab97983..a776158f1 100644 --- a/pkg/repository/prisma/workflow_run.go +++ b/pkg/repository/prisma/workflow_run.go @@ -121,6 +121,19 @@ type updateWorkflowRunQueueData struct { Event *repository.CreateStepRunEventOpts `json:"event,omitempty"` } +func (w *workflowRunEngineRepository) QueuePausedWorkflowRun(ctx context.Context, tenantId, workflowId, workflowRunId string) error { + return insertPausedWorkflowRunQueueItem( + ctx, + w.pool, + w.queries, + tenantId, + unpauseWorkflowRunQueueData{ + WorkflowId: workflowId, + WorkflowRunId: workflowRunId, + }, + ) +} + func (w *workflowRunEngineRepository) ProcessWorkflowRunUpdates(ctx context.Context, tenantId string) (bool, error) { ctx, span := telemetry.NewSpan(ctx, "process-workflow-run-updates-database") defer span.End() @@ -228,6 +241,116 @@ func (w *workflowRunEngineRepository) ProcessWorkflowRunUpdates(ctx context.Cont return len(queueItems) == limit, nil } +type unpauseWorkflowRunQueueData struct { + // NOTE: do not change this workflow_id without also changing HandleWorkflowUnpaused, + // as we've written a query which selects on this field + WorkflowId string `json:"workflow_id"` + WorkflowRunId string `json:"workflow_run_id"` +} + +func (w *workflowRunEngineRepository) ProcessUnpausedWorkflowRuns(ctx context.Context, tenantId string) ([]*dbsqlc.GetWorkflowRunRow, bool, error) { + ctx, span := telemetry.NewSpan(ctx, "process-workflow-run-updates-database") + defer span.End() + + pgTenantId := sqlchelpers.UUIDFromStr(tenantId) + + limit := 1000 + + tx, commit, rollback, err := prepareTx(ctx, w.pool, w.l, 25000) + + if err != nil { + return nil, false, err + } + + defer rollback() + + // list queues + queueItems, err := w.queries.ListInternalQueueItems(ctx, tx, dbsqlc.ListInternalQueueItemsParams{ + Tenantid: pgTenantId, + Queue: dbsqlc.InternalQueueWORKFLOWRUNPAUSED, + Limit: pgtype.Int4{ + Int32: int32(limit), + Valid: true, + }, + }) + + if err != nil { + return nil, false, fmt.Errorf("could not list internal queue items for paused workflow runs: %w", err) + } + + if len(queueItems) == 0 { + return nil, false, nil + } + + data, err := toQueueItemData[unpauseWorkflowRunQueueData](queueItems) + + if err != nil { + return nil, false, fmt.Errorf("could not convert internal queue item data to worker semaphore queue data: %w", err) + } + + // construct a map of workflow IDs + candidateUnpausedWorkflows := make(map[string]bool) + + for _, item := range data { + candidateUnpausedWorkflows[item.WorkflowId] = true + } + + // list paused workflows + pausedWorkflowIds, err := w.queries.ListPausedWorkflows(ctx, tx, sqlchelpers.UUIDFromStr(tenantId)) + + if err != nil { + return nil, false, fmt.Errorf("could not list paused workflows: %w", err) + } + + // for each workflow ID, check whether it is paused + for _, pausedWorkflowId := range pausedWorkflowIds { + delete(candidateUnpausedWorkflows, sqlchelpers.UUIDToStr(pausedWorkflowId)) + } + + // if there are no paused workflows to unpause, return + if len(candidateUnpausedWorkflows) == 0 { + return nil, false, nil + } + + // if there are paused workflows to unpause, queue them + workflowRunsToQueue := make([]pgtype.UUID, 0) + qiIds := make([]int64, 0) + + for i, item := range data { + if _, ok := candidateUnpausedWorkflows[item.WorkflowId]; ok { + workflowRunsToQueue = append(workflowRunsToQueue, sqlchelpers.UUIDFromStr(item.WorkflowRunId)) + qiIds = append(qiIds, queueItems[i].ID) + } + } + + // update the processed semaphore queue items for the workflow runs which were unpaused + err = w.queries.MarkInternalQueueItemsProcessed(ctx, tx, qiIds) + + if err != nil { + return nil, false, fmt.Errorf("could not mark worker semaphore queue items processed: %w", err) + } + + // get the workflow runs by id + workflowRuns, err := w.queries.GetWorkflowRun(ctx, tx, dbsqlc.GetWorkflowRunParams{ + Ids: workflowRunsToQueue, + Tenantid: pgTenantId, + }) + + if err != nil { + return nil, false, fmt.Errorf("could not get workflow runs by id: %w", err) + } + + err = commit(ctx) + + if err != nil { + return nil, false, fmt.Errorf("could not commit transaction: %w", err) + } + + // if we reached this point, it means that some of the workflows in the queue were unpaused, so + // we should continue until this is no longer true + return workflowRuns, true, nil +} + func (w *workflowRunAPIRepository) GetWorkflowRunById(ctx context.Context, tenantId, id string) (*dbsqlc.GetWorkflowRunByIdRow, error) { return w.queries.GetWorkflowRunById(ctx, w.pool, dbsqlc.GetWorkflowRunByIdParams{ Tenantid: sqlchelpers.UUIDFromStr(tenantId), @@ -1269,6 +1392,26 @@ func insertWorkflowRunQueueItem( ) } +func insertPausedWorkflowRunQueueItem( + ctx context.Context, + dbtx dbsqlc.DBTX, + queries *dbsqlc.Queries, + tenantId string, + data unpauseWorkflowRunQueueData, +) error { + insertData := make([]any, 1) + insertData[0] = data + + return bulkInsertInternalQueueItem( + ctx, + dbtx, + queries, + tenantId, + dbsqlc.InternalQueueWORKFLOWRUNPAUSED, + insertData, + ) +} + func bulkWorkflowRunEvents( ctx context.Context, l *zerolog.Logger, diff --git a/pkg/repository/workflow.go b/pkg/repository/workflow.go index d0191defe..7e1f4ddad 100644 --- a/pkg/repository/workflow.go +++ b/pkg/repository/workflow.go @@ -233,6 +233,11 @@ type GetWorkflowMetricsOpts struct { Status *string `validate:"omitnil,oneof=PENDING QUEUED RUNNING SUCCEEDED FAILED"` } +type UpdateWorkflowOpts struct { + // (optional) is paused -- if true, the workflow will not be scheduled + IsPaused *bool +} + type WorkflowAPIRepository interface { // ListWorkflows returns all workflows for a given tenant. ListWorkflows(tenantId string, opts *ListWorkflowsOpts) (*ListWorkflowsResult, error) @@ -254,6 +259,9 @@ type WorkflowAPIRepository interface { // GetWorkflowVersionMetrics returns the metrics for a given workflow version. GetWorkflowMetrics(tenantId, workflowId string, opts *GetWorkflowMetricsOpts) (*WorkflowMetrics, error) + // UpdateWorkflow updates a workflow for a given tenant. + UpdateWorkflow(ctx context.Context, tenantId, workflowId string, opts *UpdateWorkflowOpts) (*dbsqlc.Workflow, error) + // GetWorkflowWorkerCount returns the number of workers for a given workflow. GetWorkflowWorkerCount(tenantId, workflowId string) (int, int, error) } diff --git a/pkg/repository/workflow_run.go b/pkg/repository/workflow_run.go index fe38454a8..8a78d9380 100644 --- a/pkg/repository/workflow_run.go +++ b/pkg/repository/workflow_run.go @@ -448,6 +448,10 @@ type WorkflowRunEngineRepository interface { // GetWorkflowRunById returns a workflow run by id. GetWorkflowRunById(ctx context.Context, tenantId, runId string) (*dbsqlc.GetWorkflowRunRow, error) + QueuePausedWorkflowRun(ctx context.Context, tenantId, workflowId, workflowRunId string) error + + ProcessUnpausedWorkflowRuns(ctx context.Context, tenantId string) ([]*dbsqlc.GetWorkflowRunRow, bool, error) + GetWorkflowRunAdditionalMeta(ctx context.Context, tenantId, workflowRunId string) (*dbsqlc.GetWorkflowRunAdditionalMetaRow, error) ReplayWorkflowRun(ctx context.Context, tenantId, workflowRunId string) (*dbsqlc.GetWorkflowRunRow, error) diff --git a/prisma/migrations/20240928144309_v0_48_0/migration.sql b/prisma/migrations/20240928144309_v0_48_0/migration.sql new file mode 100644 index 000000000..6be078a30 --- /dev/null +++ b/prisma/migrations/20240928144309_v0_48_0/migration.sql @@ -0,0 +1,5 @@ +-- AlterEnum +ALTER TYPE "InternalQueue" ADD VALUE 'WORKFLOW_RUN_PAUSED'; + +-- AlterTable +ALTER TABLE "Workflow" ADD COLUMN "isPaused" BOOLEAN DEFAULT false; diff --git a/prisma/schema.prisma b/prisma/schema.prisma index 47538d74c..84def879a 100644 --- a/prisma/schema.prisma +++ b/prisma/schema.prisma @@ -463,6 +463,8 @@ model Workflow { // the parent tenant tenantId String @db.Uuid + isPaused Boolean? @default(false) + // the workflow name name String @@ -1347,6 +1349,7 @@ enum InternalQueue { WORKER_SEMAPHORE_COUNT STEP_RUN_UPDATE WORKFLOW_RUN_UPDATE + WORKFLOW_RUN_PAUSED } model InternalQueueItem { diff --git a/sql/migrations/20240928144316_v0.48.0.sql b/sql/migrations/20240928144316_v0.48.0.sql new file mode 100644 index 000000000..21d90380c --- /dev/null +++ b/sql/migrations/20240928144316_v0.48.0.sql @@ -0,0 +1,4 @@ +-- Add value to enum type: "InternalQueue" +ALTER TYPE "InternalQueue" ADD VALUE 'WORKFLOW_RUN_PAUSED'; +-- Modify "Workflow" table +ALTER TABLE "Workflow" ADD COLUMN "isPaused" boolean NULL DEFAULT false; diff --git a/sql/migrations/atlas.sum b/sql/migrations/atlas.sum index b866f2e7c..f75cb9ce0 100644 --- a/sql/migrations/atlas.sum +++ b/sql/migrations/atlas.sum @@ -1,4 +1,4 @@ -h1:lF5HY8eyRauexSPXlJPoY2e6Vu064xoJ2h5WzA2nkRI= +h1:ANl/rxmEbmeYdm3Tb2NXC15PisMM+/XP92yl/cspTK0= 20240115180414_init.sql h1:Ef3ZyjAHkmJPdGF/dEWCahbwgcg6uGJKnDxW2JCRi2k= 20240122014727_v0_6_0.sql h1:o/LdlteAeFgoHJ3e/M4Xnghqt9826IE/Y/h0q95Acuo= 20240126235456_v0_7_0.sql h1:KiVzt/hXgQ6esbdC6OMJOOWuYEXmy1yeCpmsVAHTFKs= @@ -60,3 +60,4 @@ h1:lF5HY8eyRauexSPXlJPoY2e6Vu064xoJ2h5WzA2nkRI= 20240923124809_v0.45.4.sql h1:SqSSiWebWGiDg978uLwiaWfcJkR5tm+8HQZOmCiOABQ= 20240926210650_v0.47.0.sql h1:Wm6QJ7GIFIsM9eym6e+PB3g/C9mM3tUoogxiBFD3E0w= 20240927172935_v0.47.1.sql h1:UlcoRDWx5xCoNjcYTjBsWkdbFZSuVJWlWZOzhTq8u/Y= +20240928144316_v0.48.0.sql h1:TX7/rN8ghoysQXy2OAMX+LdSIEA6tbdd4BZfL9wtxFQ= diff --git a/sql/schema/schema.sql b/sql/schema/schema.sql index e315329bb..4c362d8a9 100644 --- a/sql/schema/schema.sql +++ b/sql/schema/schema.sql @@ -2,7 +2,7 @@ CREATE TYPE "ConcurrencyLimitStrategy" AS ENUM ('CANCEL_IN_PROGRESS', 'DROP_NEWEST', 'QUEUE_NEWEST', 'GROUP_ROUND_ROBIN'); -- CreateEnum -CREATE TYPE "InternalQueue" AS ENUM ('WORKER_SEMAPHORE_COUNT', 'STEP_RUN_UPDATE', 'WORKFLOW_RUN_UPDATE'); +CREATE TYPE "InternalQueue" AS ENUM ('WORKER_SEMAPHORE_COUNT', 'STEP_RUN_UPDATE', 'WORKFLOW_RUN_UPDATE', 'WORKFLOW_RUN_PAUSED'); -- CreateEnum CREATE TYPE "InviteLinkStatus" AS ENUM ('PENDING', 'ACCEPTED', 'REJECTED'); @@ -752,6 +752,7 @@ CREATE TABLE "Workflow" ( "tenantId" UUID NOT NULL, "name" TEXT NOT NULL, "description" TEXT, + "isPaused" BOOLEAN DEFAULT false, CONSTRAINT "Workflow_pkey" PRIMARY KEY ("id") );