diff --git a/docs/news.rst b/docs/news.rst index 24276dbb..f4bd1625 100644 --- a/docs/news.rst +++ b/docs/news.rst @@ -19,6 +19,11 @@ Changed - Add a confirmation dialog to the Cancel button. +Fixed +~~~~~ + +- When managing multiple projects, the next pending job for all but one project was unreported by the :ref:`daemonstatus.json` and :ref:`listjobs.json` webservices, and was not cancellable by the :ref:`cancel.json` webservice. + 1.4.3 (2023-09-25) ------------------ diff --git a/scrapyd/interfaces.py b/scrapyd/interfaces.py index 7c9d1c8a..18fcccc1 100644 --- a/scrapyd/interfaces.py +++ b/scrapyd/interfaces.py @@ -46,6 +46,9 @@ def next(): - a unique identifier for this run in the `_job` key This message will be passed later to IEnvironment.get_environment(). + + Called ``max_proc`` times when the launcher starts, and each time a + Scrapy process ends. """ def update_projects(): diff --git a/scrapyd/poller.py b/scrapyd/poller.py index 28cbe0ff..b54041ec 100644 --- a/scrapyd/poller.py +++ b/scrapyd/poller.py @@ -15,16 +15,22 @@ def __init__(self, config): @inlineCallbacks def poll(self): - if not self.dq.waiting: - return for project, queue in self.queues.items(): + # If the "waiting" backlog is empty (that is, if the maximum number of Scrapy processes are running): + if not self.dq.waiting: + return count = yield maybeDeferred(queue.count) if count: message = yield maybeDeferred(queue.pop) - if message is not None: # In case of a concurrently accessed queue + # The message can be None if, for example, two Scrapyd instances share a spider queue database. + if message is not None: + # Pop a dummy item from the "waiting" backlog. and fire the message. returnValue(self.dq.put(self._message(message, project))) def next(self): + """ + Add a dummy item to the "waiting" backlog (based on Twisted's implementation of DeferredQueue). + """ return self.dq.get() def update_projects(self):