Fix recent changes to Runner for Windows

docs/news.rst

@@ -25,7 +25,6 @@ Documentation
 Changed
 ~~~~~~~
 
-- **BACKWARDS-INCOMPATIBLE CHANGE:** The ``IEggStorage.get()`` interface returns a ``(version, filename)`` tuple, instead of a ``(version, file)`` tuple.
 - Drop support for end-of-life Python version 3.7.
 
 Web UI

scrapyd/eggstorage.py

@@ -29,7 +29,7 @@ def get(self, project, version=None):
                 version = self.list(project)[-1]
             except IndexError:
                 return None, None
-        return version, self._eggpath(project, version)
+        return version, open(self._eggpath(project, version), 'rb')
 
     def list(self, project):
         versions = [

scrapyd/interfaces.py

@@ -9,7 +9,7 @@ def put(eggfile, project, version):
         version"""
 
     def get(project, version=None):
-        """Return a tuple (version, filename) for the egg matching the specified
+        """Return a tuple (version, file) for the egg matching the specified
         project and version. If version is None, the latest version is
         returned. If no egg is found for the given project/version (None, None)
         should be returned."""

scrapyd/runner.py

@@ -1,5 +1,7 @@
 import os
+import shutil
 import sys
+import tempfile
 from contextlib import contextmanager
 
 from scrapy.utils.misc import load_object
@@ -18,12 +20,27 @@ def project_environment(project):
     eggstorage = eggstorage_cls(config)
 
     eggversion = os.environ.get('SCRAPYD_EGG_VERSION', None)
-    version, eggpath = eggstorage.get(project, eggversion)
-    if eggpath:
-        activate_egg(eggpath)
-
-    assert 'scrapy.conf' not in sys.modules, "Scrapy settings already loaded"
-    yield
+    version, egg = eggstorage.get(project, eggversion)
+
+    tmp = None
+    if egg:
+        try:
+            if hasattr(egg, 'name'):  # for example, FileIO
+                activate_egg(egg.name)
+            else:  # for example, BytesIO
+                tmp = tempfile.NamedTemporaryFile(suffix='.egg', prefix=f'{project}-{version}-', delete=False)
+                shutil.copyfileobj(egg, tmp)
+                tmp.close()
+                activate_egg(tmp.name)
+        finally:
+            egg.close()
+
+    try:
+        assert 'scrapy.conf' not in sys.modules, "Scrapy settings already loaded"
+        yield
+    finally:
+        if tmp:
+            os.remove(tmp.name)
 
 
 def main():

tests/test_eggstorage.py

@@ -78,20 +78,26 @@ def test_put_get_list_delete(self):
         ])
         self.assertEqual(self.eggst.list('mybot2'), [])
 
-        v, name = self.eggst.get('mybot')
-        self.assertEqual(v, "03_ver")
-        with open(name, 'rb') as f:
+        v, f = self.eggst.get('mybot')
+        try:
+            self.assertEqual(v, "03_ver")
             self.assertEqual(f.read(), b"egg03")
+        finally:
+            f.close()
 
-        v, name = self.eggst.get('mybot', '02_my branch')
-        self.assertEqual(v, "02_my branch")
-        with open(name, 'rb') as f:
+        v, f = self.eggst.get('mybot', '02_my branch')
+        try:
+            self.assertEqual(v, "02_my branch")
             self.assertEqual(f.read(), b"egg02")
+        finally:
+            f.close()
 
-        v, name = self.eggst.get('mybot', '02_my_branch')
-        self.assertEqual(v, "02_my_branch")
-        with open(name, 'rb') as f:
+        v, f = self.eggst.get('mybot', '02_my_branch')
+        try:
+            self.assertEqual(v, "02_my_branch")
             self.assertEqual(f.read(), b"egg02")
+        finally:
+            f.close()
 
         self.eggst.delete('mybot', '02_my branch')
         self.assertEqual(self.eggst.list('mybot'), ['01', '03_ver'])

tests/test_webservice.py

@@ -79,15 +79,19 @@ def test_delete_version(self, txrequest, site_with_egg):
         }
 
         storage = site_with_egg.app.getComponent(IEggStorage)
-        egg = storage.get('quotesbot')
+        version, egg = storage.get('quotesbot')
+        if egg:
+            egg.close()
+
         content = site_with_egg.children[endpoint].render_POST(txrequest)
-        no_egg = storage.get('quotesbot')
+        no_version, no_egg = storage.get('quotesbot')
+        if no_egg:
+            no_egg.close()
 
-        assert egg[0] is not None
+        assert version is not None
         assert content['status'] == 'ok'
         assert 'node_name' in content
-        assert storage.get('quotesbot')
-        assert no_egg[0] is None
+        assert no_version is None
 
     def test_delete_project(self, txrequest, site_with_egg):
         endpoint = b'delproject.json'
@@ -96,15 +100,19 @@ def test_delete_project(self, txrequest, site_with_egg):
         }
 
         storage = site_with_egg.app.getComponent(IEggStorage)
-        egg = storage.get('quotesbot')
+        version, egg = storage.get('quotesbot')
+        if egg:
+            egg.close()
+
         content = site_with_egg.children[endpoint].render_POST(txrequest)
-        no_egg = storage.get('quotesbot')
+        no_version, no_egg = storage.get('quotesbot')
+        if no_egg:
+            no_egg.close()
 
-        assert egg[0] is not None
+        assert version is not None
         assert content['status'] == 'ok'
         assert 'node_name' in content
-        assert storage.get('quotesbot')
-        assert no_egg[0] is None
+        assert no_version is None
 
     @mock.patch('scrapyd.webservice.get_spider_list', new=fake_list_spiders)
     def test_addversion(self, txrequest, site_no_egg):
@@ -118,15 +126,19 @@ def test_addversion(self, txrequest, site_no_egg):
             txrequest.args[b'egg'] = [f.read()]
 
         storage = site_no_egg.app.getComponent(IEggStorage)
-        egg = storage.get('quotesbot')
+        version, egg = storage.get('quotesbot')
+        if egg:
+            egg.close()
+
         content = site_no_egg.children[endpoint].render_POST(txrequest)
-        no_egg = storage.get('quotesbot')
+        no_version, no_egg = storage.get('quotesbot')
+        if no_egg:
+            no_egg.close()
 
-        assert egg[0] is None
+        assert version is None
         assert content['status'] == 'ok'
         assert 'node_name' in content
-        assert storage.get('quotesbot')
-        assert no_egg[0] == '0_1'
+        assert no_version == '0_1'
 
     @mock.patch('scrapyd.webservice.get_spider_list',
                 new=fake_list_spiders_other)