master: abort jobs when the quota is exceeded, #TASK-6446

opencb · Nov 5, 2024 · 484e763 · 484e763
1 parent fe6f97d
commit 484e763
Show file tree

Hide file tree

Showing 2 changed files with 18 additions and 6 deletions.
diff --git a/opencga-catalog/src/main/java/org/opencb/opencga/catalog/managers/JobManager.java b/opencga-catalog/src/main/java/org/opencb/opencga/catalog/managers/JobManager.java
@@ -30,9 +30,7 @@
 import org.opencb.opencga.catalog.auth.authorization.AuthorizationManager;
 import org.opencb.opencga.catalog.db.DBAdaptorFactory;
 import org.opencb.opencga.catalog.db.api.*;
-import org.opencb.opencga.catalog.exceptions.CatalogAuthorizationException;
-import org.opencb.opencga.catalog.exceptions.CatalogException;
-import org.opencb.opencga.catalog.exceptions.CatalogIOException;
+import org.opencb.opencga.catalog.exceptions.*;
 import org.opencb.opencga.catalog.io.IOManager;
 import org.opencb.opencga.catalog.io.IOManagerFactory;
 import org.opencb.opencga.catalog.models.InternalGetDataResult;
@@ -623,17 +621,24 @@ public OpenCGAResult<ExecutionTime> getExecutionTimeByMonth(String organizationI
     }
 
     private void checkExecutionLimitQuota(String organizationId) throws CatalogException {
+        if (exceedsExecutionLimitQuota(organizationId)) {
+            throw new CatalogException("The organization '" + organizationId + "' has reached the maximum quota of execution hours ("
+                    + configuration.getQuota().getMaxNumJobHours() + ") for the current month.");
+        }
+    }
+
+    public boolean exceedsExecutionLimitQuota(String organizationId) throws CatalogException {
         // Get current year/month
         String time = TimeUtils.getTime(TimeUtils.getDate(), TimeUtils.yyyyMM);
         Query query = new Query(JobDBAdaptor.QueryParams.MODIFICATION_DATE.key(), time);
         OpenCGAResult<ExecutionTime> result = getJobDBAdaptor(organizationId).executionTimeByMonth(query);
         if (result.getNumResults() > 0) {
             ExecutionTime executionTime = result.first();
             if (executionTime.getTime().getHours() >= configuration.getQuota().getMaxNumJobHours()) {
-                throw new CatalogException("The organization '" + organizationId + "' has reached the maximum quota of execution hours ("
-                        + configuration.getQuota().getMaxNumJobHours() + ") for the current month.");
+                return true;
             }
         }
+        return false;
     }
 
     /**

diff --git a/opencga-master/src/main/java/org/opencb/opencga/master/monitor/daemons/ExecutionDaemon.java b/opencga-master/src/main/java/org/opencb/opencga/master/monitor/daemons/ExecutionDaemon.java
@@ -500,8 +500,15 @@ protected void checkPendingJobs(List<String> organizationIds) {
 
         for (String organizationId : organizationIds) {
             try (DBIterator<Job> iterator = jobManager.iteratorInOrganization(organizationId, pendingJobsQuery, queryOptions, token)) {
+                boolean exceeds = jobManager.exceedsExecutionLimitQuota(organizationId);
                 while (iterator.hasNext()) {
-                    pendingJobs.add(iterator.next());
+                    if (exceeds) {
+                        abortJob(iterator.next(), "The organization '" + organizationId + "' has reached the maximum limit quota of"
+                                + " execution hours (" + catalogManager.getConfiguration().getQuota().getMaxNumJobHours()
+                                + ") for the current month.");
+                    } else {
+                        pendingJobs.add(iterator.next());
+                    }
                 }
             } catch (Exception e) {
                 logger.error("Error listing pending jobs from organization {}", organizationId, e);