From 7fd516c9e77a903b82b9fef65d0cdfa52e6b1f7a Mon Sep 17 00:00:00 2001 From: Blondel MONDESIR Date: Sat, 6 Jul 2024 17:36:10 -0400 Subject: [PATCH 1/4] Use timezone-aware objects --- cps/services/worker.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/cps/services/worker.py b/cps/services/worker.py index 2c57689f9..cca65db88 100644 --- a/cps/services/worker.py +++ b/cps/services/worker.py @@ -25,7 +25,7 @@ import queue except ImportError: import Queue as queue -from datetime import datetime +from datetime import datetime, timezone from collections import namedtuple from cps import logger @@ -92,7 +92,7 @@ def add(cls, user, task, hidden=False): ins.queue.put(QueuedTask( num=ins.num, user=username, - added=datetime.now(), + added=datetime.now(timezone.utc), task=task, hidden=hidden )) @@ -191,7 +191,7 @@ def is_cancellable(self): raise NotImplementedError def start(self, *args): - self.start_time = datetime.now() + self.start_time = datetime.now(timezone.utc) self.stat = STAT_STARTED # catch any unhandled exceptions in a task and automatically fail it @@ -201,7 +201,7 @@ def start(self, *args): self._handleError(str(ex)) log.error_or_exception(ex) - self.end_time = datetime.now() + self.end_time = datetime.now(timezone.utc) @property def stat(self): @@ -231,7 +231,8 @@ def error(self, x): @property def runtime(self): - return (self.end_time or datetime.now()) - self.start_time + now = datetime.now(timezone.utc) + return (self.end_time or now) - self.start_time @property def dead(self): From fd79ef0d0f3a6382a6bc7bae55aeca2c364d6352 Mon Sep 17 00:00:00 2001 From: Blondel MONDESIR Date: Sat, 6 Jul 2024 17:45:42 -0400 Subject: [PATCH 2/4] Make start_time and end_time timezone-aware --- cps/tasks/download.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/cps/tasks/download.py b/cps/tasks/download.py index efa2c7af2..51469695c 100644 --- a/cps/tasks/download.py +++ b/cps/tasks/download.py @@ -3,7 +3,7 @@ import requests import select import sqlite3 -from datetime import datetime +from datetime import datetime, timezone from flask_babel import lazy_gettext as N_, gettext as _ from cps.constants import XKLB_DB_FILE @@ -25,14 +25,14 @@ def __init__(self, task_message, media_url, original_url, current_user_name, she self.shelf_id = shelf_id self.duration = datetime.utcfromtimestamp(int(duration)).strftime("%H:%M:%S") if duration else "unknown" self.live_status = live_status - self.start_time = self.end_time = datetime.now() + self.start_time = self.end_time = datetime.now(timezone.utc) self.stat = STAT_WAITING self.progress = 0 def run(self, worker_thread): """Run the download task""" self.worker_thread = worker_thread - self.start_time = self.end_time = datetime.now() + self.end_time = datetime.now(timezone.utc) self.stat = STAT_STARTED self.progress = 0 @@ -53,14 +53,14 @@ def run(self, worker_thread): complete_progress_cycle = 0 - last_progress_time = datetime.now() + last_progress_time = datetime.now(timezone.utc) fragment_stuck_timeout = 30 # seconds self.message = f"Downloading {self.media_url_link}..." if self.live_status == "was_live": self.message += f" (formerly live video, length/duration {self.duration})" while p.poll() is None: - self.end_time = datetime.now() + self.end_time = datetime.now(timezone.utc) # Check if there's data available to read rlist, _, _ = select.select([p.stdout], [], [], 0.1) if rlist: @@ -77,9 +77,9 @@ def run(self, worker_thread): self.progress = min(0.99, (complete_progress_cycle + (percentage / 100)) / 4) if percentage == 100: complete_progress_cycle += 1 - last_progress_time = datetime.now() + last_progress_time = datetime.now(timezone.utc) else: - elapsed_time = (datetime.now() - last_progress_time).total_seconds() + elapsed_time = (datetime.now(timezone.utc) - last_progress_time).total_seconds() if elapsed_time >= fragment_stuck_timeout: self.message += f"
Some fragments are taking longer than expected to download. Please wait..." @@ -133,7 +133,7 @@ def run(self, worker_thread): self.message = f"{self.media_url_link} failed to download: {self.read_error_from_database()}" finally: - self.end_time = datetime.now() + self.end_time = datetime.now(timezone.utc) if p.returncode == 0 or self.progress == 1.0: self.stat = STAT_FINISH_SUCCESS log.info("Download task for %s completed successfully", self.media_url) From 9f52798b4d068bc7f804945f4a364b2650723f2c Mon Sep 17 00:00:00 2001 From: Blondel MONDESIR Date: Sat, 6 Jul 2024 17:57:58 -0400 Subject: [PATCH 3/4] Make start_time and end_time timezone-aware --- cps/tasks/metadata_extract.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/cps/tasks/metadata_extract.py b/cps/tasks/metadata_extract.py index a946790ba..e16e96f89 100644 --- a/cps/tasks/metadata_extract.py +++ b/cps/tasks/metadata_extract.py @@ -2,7 +2,7 @@ import re import requests import sqlite3 -from datetime import datetime +from datetime import datetime, timezone from flask_babel import lazy_gettext as N_, gettext as _ from cps.constants import XKLB_DB_FILE, MAX_VIDEOS_PER_DOWNLOAD @@ -23,7 +23,7 @@ def __init__(self, task_message, media_url, original_url, current_user_name): self.original_url = self._format_original_url(original_url) self.is_playlist = None self.current_user_name = current_user_name - self.start_time = self.end_time = datetime.now() + self.start_time = self.end_time = datetime.now(timezone.utc) self.stat = STAT_WAITING self.progress = 0 self.columns = None @@ -110,7 +110,7 @@ def _update_metadata(self, requested_urls): requested_urls = {url: requested_urls[url] for url in requested_urls.keys() if "shorts" not in url and url not in failed_urls} def _calculate_views_per_day(self, requested_urls, conn): - now = datetime.now() + now = datetime.now(timezone.utc) for requested_url in requested_urls.keys(): try: view_count = conn.execute("SELECT view_count FROM media WHERE path = ?", (requested_url,)).fetchone()[0] @@ -151,7 +151,7 @@ def _add_download_tasks_to_worker(self, requested_urls): def run(self, worker_thread): self.worker_thread = worker_thread log.info("Starting to fetch metadata for URL: %s", self.media_url) - self.start_time = self.end_time = datetime.now() + self.end_time = datetime.now(timezone.utc) self.stat = STAT_STARTED self.progress = 0 From c97f5505c6ae9d5dd37f99c9ed37dc71c9447e02 Mon Sep 17 00:00:00 2001 From: Blondel MONDESIR Date: Tue, 23 Jul 2024 14:37:41 -0400 Subject: [PATCH 4/4] Make last_progress_time timezone-aware --- cps/tasks/download.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cps/tasks/download.py b/cps/tasks/download.py index c18e582f7..e7e9d8797 100644 --- a/cps/tasks/download.py +++ b/cps/tasks/download.py @@ -53,7 +53,7 @@ def run(self, worker_thread): complete_progress_cycle = 0 - last_progress_time = datetime.now() + last_progress_time = datetime.now(timezone.utc) fragment_stuck_timeout = 120 # seconds self.message = f"Downloading {self.media_url_link}..."