fix: Fix #327 when managing multiple projects

docs/news.rst

@@ -19,6 +19,11 @@ Changed
 
 - Add a confirmation dialog to the Cancel button.
 
+Fixed
+~~~~~
+
+- When managing multiple projects, the next pending job for all but one project was unreported by the :ref:`daemonstatus.json` and :ref:`listjobs.json` webservices, and was not cancellable by the :ref:`cancel.json` webservice.
+
 1.4.3 (2023-09-25)
 ------------------
 

scrapyd/interfaces.py

@@ -46,6 +46,9 @@ def next():
         -  a unique identifier for this run in the `_job` key
 
         This message will be passed later to IEnvironment.get_environment().
+
+        Called ``max_proc`` times when the launcher starts, and each time a
+        Scrapy process ends.
         """
 
     def update_projects():

scrapyd/poller.py

@@ -15,16 +15,22 @@ def __init__(self, config):
 
     @inlineCallbacks
     def poll(self):
-        if not self.dq.waiting:
-            return
         for project, queue in self.queues.items():
+            # If the "waiting" backlog is empty (that is, if the maximum number of Scrapy processes are running):
+            if not self.dq.waiting:
+                return
             count = yield maybeDeferred(queue.count)
             if count:
                 message = yield maybeDeferred(queue.pop)
-                if message is not None:  # In case of a concurrently accessed queue
+                # The message can be None if, for example, two Scrapyd instances share a spider queue database.
+                if message is not None:
+                    # Pop a dummy item from the "waiting" backlog. and fire the message.
                     returnValue(self.dq.put(self._message(message, project)))
 
     def next(self):
+        """
+        Add a dummy item to the "waiting" backlog (based on Twisted's implementation of DeferredQueue).
+        """
         return self.dq.get()
 
     def update_projects(self):