Run jobs in 2hr chunks

luomus · Aug 7, 2024 · 2a2b038 · 2a2b038
1 parent 832051e
commit 2a2b038
Show file tree

Hide file tree

Showing 3 changed files with 10 additions and 12 deletions.
diff --git a/oc-process.sh b/oc-process.sh
@@ -24,7 +24,7 @@ if [ "$BRANCH" != "main" ]; then
 
   HOST=$HOST_DEV
   DB_PASSWORD=$DB_PASSWORD_DEV
-  TIMEOUT_IN_HOURS=$TIMEOUT_IN_HOURS_DEV
+  SCHEDULE=$SCHEDULE_DEV
   JOB_SECRET=$JOB_SECRET_DEV
   STORAGE=$STORAGE_DEV
   MEMORY=$MEMORY_DEV
@@ -95,7 +95,7 @@ oc process -f $f \
   -p FINBIF_API="$FINBIF_API" \
   -p FINBIF_WAREHOUSE_QUERY="$FINBIF_WAREHOUSE_QUERY" \
   -p FINBIF_EMAIL="$FINBIF_EMAIL" \
-  -p TIMEOUT_IN_HOURS="$TIMEOUT_IN_HOURS" \
+  -p SCHEDULE="$SCHEDULE" \
   -p SMTP_SERVER="$SMTP_SERVER" \
   -p SMTP_PORT="$SMTP_PORT" \
   -p ERROR_EMAIL_FROM="$ERROR_EMAIL_FROM" \

diff --git a/template.yml b/template.yml
@@ -64,8 +64,6 @@ parameters:
     required: true
   - name: FINBIF_EMAIL
     required: true
-  - name: TIMEOUT_IN_HOURS
-    required: true
   - name: SMTP_SERVER
     required: true
   - name: SMTP_PORT
@@ -82,6 +80,8 @@ parameters:
     required: true
   - name: MEMORY
     required: true
+  - name: SCHEDULE
+    required: true
 metadata:
   name: ${APP}
 objects:
@@ -362,13 +362,13 @@ objects:
   metadata:
     name: ${APP}-${BRANCH}
   spec:
-    schedule: "0 20 * * *"
+    schedule: ${SCHEDULE}
     concurrencyPolicy: Forbid
-    successfulJobsHistoryLimit: 7
-    failedJobsHistoryLimit: 7
+    successfulJobsHistoryLimit: 10
+    failedJobsHistoryLimit: 10
     jobTemplate:
       spec:
-        activeDeadlineSeconds: 36000
+        activeDeadlineSeconds: 7200
         backoffLimit: 0
         template:
           metadata:
@@ -424,8 +424,6 @@ objects:
                   configMapKeyRef:
                     name: ${APP}-${BRANCH}
                     key: db_host
-              - name: TIMEOUT_IN_HOURS
-                value: ${TIMEOUT_IN_HOURS}
               - name: JOB_SECRET
                 valueFrom:
                   secretKeyRef:

diff --git a/update_indices.R b/update_indices.R
@@ -1,6 +1,6 @@
 dir.create("var/logs", showWarnings = FALSE)
 
-log_file_name <- sprintf("var/logs/update-%s.txt", Sys.Date())
+log_file_name <- sprintf("var/logs/update-%s.txt", format(Sys.time(), "%F-%H"))
 
 log_file <- file(log_file_name, open = "wt")
 
@@ -51,7 +51,7 @@ res <- withCallingHandlers(
 
       }
 
-      timeout_in_secs <- as.integer(Sys.getenv("TIMEOUT_IN_HOURS")) * 60L * 60L
+      timeout_in_secs <- 6600L
 
       start_timer <- tictoc::tic()