chore: Improve app readability

scrapy · Jul 21, 2024 · 51b3fc4 · 51b3fc4
1 parent 261f0b9
commit 51b3fc4
Show file tree

Hide file tree

Showing 3 changed files with 14 additions and 11 deletions.
diff --git a/scrapyd/app.py b/scrapyd/app.py
@@ -19,28 +19,26 @@ def application(config):
     unix_socket_path = os.getenv("SCRAPYD_UNIX_SOCKET_PATH") or config.get("unix_socket_path", "")
     poll_interval = config.getfloat("poll_interval", 5)
 
+    environment = Environment(config)
     scheduler = SpiderScheduler(config)
-    app.setComponent(ISpiderScheduler, scheduler)
+    poller = initialize_component(config, "poller", "scrapyd.poller.QueuePoller")
+    jobstorage = initialize_component(config, "jobstorage", "scrapyd.jobstorage.MemoryJobStorage")
+    eggstorage = initialize_component(config, "eggstorage", "scrapyd.eggstorage.FilesystemEggStorage")
 
-    environment = Environment(config)
     app.setComponent(IEnvironment, environment)
-
-    poller = initialize_component(config, "poller", "scrapyd.poller.QueuePoller")
+    app.setComponent(ISpiderScheduler, scheduler)
     app.setComponent(IPoller, poller)
-
-    jobstorage = initialize_component(config, "jobstorage", "scrapyd.jobstorage.MemoryJobStorage")
     app.setComponent(IJobStorage, jobstorage)
-
-    eggstorage = initialize_component(config, "eggstorage", "scrapyd.eggstorage.FilesystemEggStorage")
     app.setComponent(IEggStorage, eggstorage)
 
+    # launcher uses jobstorage in initializer, and uses poller and environment.
     launcher = initialize_component(config, "launcher", "scrapyd.launcher.Launcher", app)
 
     timer = TimerService(poll_interval, poller.poll)
 
+    # webroot uses launcher, poller, scheduler and environment.
     webroot = initialize_component(config, "webroot", "scrapyd.website.Root", app)
     resource = server.Site(wrap_resource(webroot, config))
-
     if bind_address and http_port:
         webservice = TCPServer(http_port, resource, interface=bind_address)
         log.msg(

diff --git a/scrapyd/runner.py b/scrapyd/runner.py
@@ -35,15 +35,17 @@ def project_environment(project):
     eggstorage = initialize_component(config, "eggstorage", "scrapyd.eggstorage.FilesystemEggStorage")
 
     eggversion = os.environ.get("SCRAPYD_EGG_VERSION", None)
-    version, egg = eggstorage.get(project, eggversion)
+    sanitized_version, egg = eggstorage.get(project, eggversion)
 
     tmp = None
+    # egg can be None if the project is not in eggstorage, but is defined in the [settings] configuration section.
     if egg:
         try:
             if hasattr(egg, "name"):  # for example, FileIO
                 activate_egg(egg.name)
             else:  # for example, BytesIO
-                tmp = tempfile.NamedTemporaryFile(suffix=".egg", prefix=f"{project}-{version}-", delete=False)
+                prefix = f"{project}-{sanitized_version}-"
+                tmp = tempfile.NamedTemporaryFile(suffix=".egg", prefix=prefix, delete=False)
                 shutil.copyfileobj(egg, tmp)
                 tmp.close()
                 activate_egg(tmp.name)

diff --git a/scrapyd/utils.py b/scrapyd/utils.py
@@ -41,6 +41,9 @@ def get_project_list(config):
     """Get list of projects by inspecting the eggs storage and the ones defined in
     the scrapyd.conf [settings] section
     """
+
+    # The poller and scheduler use this function (via get_spider_queues), and they aren't initialized with the
+    # application. So, we need to re-initialize this component here.
     eggstorage = initialize_component(config, "eggstorage", "scrapyd.eggstorage.FilesystemEggStorage")
     projects = eggstorage.list_projects()
     projects.extend(project for project, _ in config.items("settings", default=[]))