chore: Remove native_stringify_dict

docs/news.rst

@@ -115,8 +115,8 @@ Scrapyd is now tested on macOS and Windows, in addition to Linux.
 Removed
 ~~~~~~~
 
--  Remove the unused keyword arguments from the ``native_stringify_dict`` function.
 -  Remove the ``JsonSqliteDict`` and ``UtilsCache`` classes.
+-  Remove the ``native_stringify_dict`` function.
 -  Remove undocumented and unused internal environment variables:
 
   - ``SCRAPY_FEED_URI`` to ``SCRAPYD_FEED_URI``

scrapyd/launcher.py

@@ -9,7 +9,6 @@
 
 from scrapyd import __version__
 from scrapyd.interfaces import IEnvironment, IJobStorage, IPoller
-from scrapyd.utils import native_stringify_dict
 
 log = Logger()
 
@@ -54,19 +53,15 @@ def _get_message(self, slot):
         poller.next().addCallback(self._spawn_process, slot)
 
     def _spawn_process(self, message, slot):
+        project = message["_project"]
         environ = self.app.getComponent(IEnvironment)
         message.setdefault("settings", {})
         message["settings"].update(environ.get_settings(message))
-        decoded = native_stringify_dict(message)
-        project = decoded["_project"]
 
-        args = [sys.executable, "-m", self.runner, "crawl"]
-        args += get_crawl_args(decoded)
+        env = environ.get_environment(message, slot)
+        args = [sys.executable, "-m", self.runner, "crawl", *get_crawl_args(message)]
 
-        env = environ.get_environment(decoded, slot)
-        env = native_stringify_dict(env)
-
-        process = ScrapyProcessProtocol(project, decoded["_spider"], decoded["_job"], env, args)
+        process = ScrapyProcessProtocol(project, message["_spider"], message["_job"], env, args)
         process.deferred.addBoth(self._process_finished, slot)
 
         reactor.spawnProcess(process, sys.executable, args=args, env=env)

scrapyd/utils.py

@@ -48,27 +48,3 @@ def get_project_list(config):
     projects = eggstorage.list_projects()
     projects.extend(project for project, _ in config.items("settings", default=[]))
     return projects
-
-
-def native_stringify_dict(dct_or_tuples):
-    """Return a (new) dict with unicode keys and values
-    of the given dict converted to strings. `dct_or_tuples` can be a
-    dict or a list of tuples, like any dict constructor supports.
-    """
-    d = {}
-    for k, v in dct_or_tuples.items():
-        key = _to_native_str(k)
-        if isinstance(v, dict):
-            value = native_stringify_dict(v)
-        elif isinstance(v, list):
-            value = [_to_native_str(e) for e in v]
-        else:
-            value = _to_native_str(v)
-        d[key] = value
-    return d
-
-
-def _to_native_str(text):
-    if isinstance(text, str):
-        return text
-    return text.decode()

scrapyd/webservice.py

@@ -8,7 +8,6 @@
 import uuid
 import zipfile
 from collections import defaultdict
-from copy import copy
 from io import BytesIO
 from subprocess import PIPE, Popen
 from typing import ClassVar
@@ -17,7 +16,6 @@
 from twisted.web import error, http, resource
 
 from scrapyd.exceptions import EggNotFoundError, ProjectNotFoundError, RunnerError
-from scrapyd.utils import native_stringify_dict
 
 log = Logger()
 
@@ -186,7 +184,7 @@ def render_POST(self, txrequest, project, spider, version, jobid, priority, sett
         if spider not in spiders:
             raise error.Error(code=http.OK, message=b"spider '%b' not found" % spider.encode())
 
-        spider_arguments = {k: v[0] for k, v in native_stringify_dict(copy(txrequest.args)).items()}
+        spider_arguments = {key.decode(): values[0].decode() for key, values in txrequest.args.items()}
 
         self.root.scheduler.schedule(
             project,

tests/test_webservice.py

@@ -155,7 +155,7 @@ def test_debug(txrequest, root):
 
     assert response.startswith("Traceback (most recent call last):")
     assert response.endswith(
-        f"twisted.web.error.Error: 200 priority is invalid: could not convert string to float: b'x'{os.linesep}"
+        "twisted.web.error.Error: 200 priority is invalid: could not convert string to float: b'x'\n"
     )