diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 791f7036..6f7c7bf0 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -67,6 +67,6 @@ jobs: - uses: actions/setup-python@v3 - name: 'Run linters' run: | - python3 -m pip install ruff rstcheck toml-sort sphinx-rtd-theme + python3 -m pip install black ruff rstcheck toml-sort sphinx-rtd-theme python3 -m pip freeze make lint-all diff --git a/HISTORY.rst b/HISTORY.rst index 3e78eff8..41d284fa 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -1,5 +1,12 @@ Bug tracker at https://github.com/giampaolo/pyftpdlib/issues +Version: 1.5.10 - (UNRELEASED) +============================== + +**Enhancements** + +* #621: use black formatter. + Version: 1.5.9 - 2023-10-25 =========================== diff --git a/Makefile b/Makefile index d300c997..4f49e606 100644 --- a/Makefile +++ b/Makefile @@ -6,6 +6,7 @@ PYTHON = python3 TSCRIPT = pyftpdlib/test/runner.py ARGS = PYDEPS = \ + black \ check-manifest \ coverage \ psutil \ @@ -152,8 +153,11 @@ test-coverage: ## Run test coverage. # Linters # =================================================================== +black: ## Python files linting (via black) + @git ls-files '*.py' | xargs $(PYTHON) -m black --check --safe + ruff: ## Run ruff linter. - @git ls-files '*.py' | xargs $(PYTHON) -m ruff check --config=pyproject.toml --no-cache + @git ls-files '*.py' | xargs $(PYTHON) -m ruff check --no-cache _pylint: ## Python pylint (not mandatory, just run it from time to time) @git ls-files '*.py' | xargs $(PYTHON) -m pylint --rcfile=pyproject.toml --jobs=${NUM_WORKERS} @@ -165,6 +169,7 @@ lint-toml: ## Linter for pyproject.toml @git ls-files '*.toml' | xargs toml-sort --check lint-all: ## Run all linters + ${MAKE} black ${MAKE} ruff ${MAKE} lint-rst ${MAKE} lint-toml @@ -173,6 +178,9 @@ lint-all: ## Run all linters # Fixers # =================================================================== +fix-black: + git ls-files '*.py' | xargs $(PYTHON) -m black + fix-ruff: @git ls-files '*.py' | xargs $(PYTHON) -m ruff --config=pyproject.toml --no-cache --fix @@ -183,6 +191,7 @@ fix-unittests: ## Fix unittest idioms. @git ls-files '*test_*.py' | xargs $(PYTHON) -m teyit --show-stats fix-all: ## Run all code fixers. + ${MAKE} fix-black ${MAKE} fix-ruff ${MAKE} fix-toml ${MAKE} fix-unittests diff --git a/demo/anti_flood_ftpd.py b/demo/anti_flood_ftpd.py index 8fc9a177..87877f7a 100755 --- a/demo/anti_flood_ftpd.py +++ b/demo/anti_flood_ftpd.py @@ -19,14 +19,15 @@ class AntiFloodHandler(FTPHandler): cmds_per_second = 300 # max number of cmds per second - ban_for = 60 * 60 # 1 hour + ban_for = 60 * 60 # 1 hour banned_ips = [] def __init__(self, *args, **kwargs): FTPHandler.__init__(self, *args, **kwargs) self.processed_cmds = 0 - self.pcmds_callback = \ - self.ioloop.call_every(1, self.check_processed_cmds) + self.pcmds_callback = self.ioloop.call_every( + 1, self.check_processed_cmds + ) def on_connect(self): # called when client connects. diff --git a/demo/tls_ftpd.py b/demo/tls_ftpd.py index 2e5a312c..0b7f3851 100755 --- a/demo/tls_ftpd.py +++ b/demo/tls_ftpd.py @@ -16,8 +16,9 @@ from pyftpdlib.servers import FTPServer -CERTFILE = os.path.abspath(os.path.join(os.path.dirname(__file__), - "keycert.pem")) +CERTFILE = os.path.abspath( + os.path.join(os.path.dirname(__file__), "keycert.pem") +) def main(): diff --git a/demo/unix_daemon.py b/demo/unix_daemon.py index 27947f91..a78075b4 100755 --- a/demo/unix_daemon.py +++ b/demo/unix_daemon.py @@ -122,6 +122,7 @@ def get_server(): def daemonize(): """A wrapper around python-daemonize context manager.""" + def _daemonize(): pid = os.fork() if pid > 0: @@ -170,10 +171,16 @@ def main(): USAGE = "python3 [-p PIDFILE] [-l LOGFILE]\n\n" USAGE += "Commands:\n - start\n - stop\n - status" parser = optparse.OptionParser(usage=USAGE) - parser.add_option('-l', '--logfile', dest='logfile', - help='the log file location') - parser.add_option('-p', '--pidfile', dest='pidfile', default=PID_FILE, - help='file to store/retreive daemon pid') + parser.add_option( + '-l', '--logfile', dest='logfile', help='the log file location' + ) + parser.add_option( + '-p', + '--pidfile', + dest='pidfile', + default=PID_FILE, + help='file to store/retreive daemon pid', + ) options, args = parser.parse_args() if options.pidfile: diff --git a/demo/unix_ftpd.py b/demo/unix_ftpd.py index 62941a9e..844694fc 100755 --- a/demo/unix_ftpd.py +++ b/demo/unix_ftpd.py @@ -17,8 +17,9 @@ def main(): - authorizer = UnixAuthorizer(rejected_users=["root"], - require_valid_shell=True) + authorizer = UnixAuthorizer( + rejected_users=["root"], require_valid_shell=True + ) handler = FTPHandler handler.authorizer = authorizer handler.abstracted_fs = UnixFilesystem diff --git a/docs/conf.py b/docs/conf.py index ca90f18b..bf9ca828 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -36,8 +36,9 @@ def get_version(): - INIT = os.path.abspath(os.path.join(HERE, '..', 'pyftpdlib', - '__init__.py')) + INIT = os.path.abspath( + os.path.join(HERE, '..', 'pyftpdlib', '__init__.py') + ) with open(INIT) as f: for line in f: if line.startswith('__ver__'): @@ -59,11 +60,13 @@ def get_version(): # Add any Sphinx extension module names here, as strings. They can be # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom # ones. -extensions = ['sphinx.ext.autodoc', - 'sphinx.ext.coverage', - 'sphinx.ext.imgmath', - 'sphinx.ext.viewcode', - 'sphinx.ext.intersphinx'] +extensions = [ + 'sphinx.ext.autodoc', + 'sphinx.ext.coverage', + 'sphinx.ext.imgmath', + 'sphinx.ext.viewcode', + 'sphinx.ext.intersphinx', +] # Add any paths that contain templates here, relative to this directory. templates_path = ['_templates'] @@ -274,15 +277,12 @@ def get_version(): # The paper size ('letterpaper' or 'a4paper'). # # 'papersize': 'letterpaper', - # The font size ('10pt', '11pt' or '12pt'). # # 'pointsize': '10pt', - # Additional stuff for the LaTeX preamble. # # 'preamble': '', - # Latex figure (float) alignment # # 'figure_align': 'htbp', @@ -292,8 +292,7 @@ def get_version(): # (source start file, target name, title, # author, documentclass [howto, manual, or own class]). latex_documents = [ - (master_doc, 'pyftpdlib.tex', 'pyftpdlib Documentation', - AUTHOR, 'manual'), + (master_doc, 'pyftpdlib.tex', 'pyftpdlib Documentation', AUTHOR, 'manual'), ] # The name of an image file (relative to this directory) to place at the top of @@ -333,10 +332,7 @@ def get_version(): # One entry per manual page. List of tuples # (source start file, name, description, authors, manual section). -man_pages = [ - (master_doc, 'pyftpdlib', 'pyftpdlib Documentation', - [author], 1) -] +man_pages = [(master_doc, 'pyftpdlib', 'pyftpdlib Documentation', [author], 1)] # If true, show URL addresses after external links. # @@ -349,9 +345,15 @@ def get_version(): # (source start file, target name, title, author, # dir menu entry, description, category) texinfo_documents = [ - (master_doc, 'pyftpdlib', 'pyftpdlib Documentation', - author, 'pyftpdlib', 'One line description of project.', - 'Miscellaneous'), + ( + master_doc, + 'pyftpdlib', + 'pyftpdlib Documentation', + author, + 'pyftpdlib', + 'One line description of project.', + 'Miscellaneous', + ), ] # Documents to append as an appendix to all manuals. diff --git a/pyftpdlib/__init__.py b/pyftpdlib/__init__.py index 9e7427ea..76ae1fbd 100644 --- a/pyftpdlib/__init__.py +++ b/pyftpdlib/__init__.py @@ -65,6 +65,7 @@ class used to interact with the file system, providing a high level, [I 13-02-19 10:56:27] 127.0.0.1:34179-[user] RETR /home/giampaolo/.vimrc completed=1 bytes=1700 seconds=0.001 [I 13-02-19 10:56:39] 127.0.0.1:34179-[user] FTP session closed (disconnect). + """ diff --git a/pyftpdlib/__main__.py b/pyftpdlib/__main__.py index fc2bb98b..55c1cc31 100644 --- a/pyftpdlib/__main__.py +++ b/pyftpdlib/__main__.py @@ -37,37 +37,92 @@ def format_option(self, option): def main(): """Start a stand alone anonymous FTP server.""" usage = "python3 -m pyftpdlib [options]" - parser = optparse.OptionParser(usage=usage, description=main.__doc__, - formatter=CustomizedOptionFormatter()) - parser.add_option('-i', '--interface', default=None, metavar="ADDRESS", - help="specify the interface to run on (default all " - "interfaces)") - parser.add_option('-p', '--port', type="int", default=2121, metavar="PORT", - help="specify port number to run on (default 2121)") - parser.add_option('-w', '--write', action="store_true", default=False, - help="grants write access for logged in user " - "(default read-only)") - parser.add_option('-d', '--directory', default=getcwdu(), metavar="FOLDER", - help="specify the directory to share (default current " - "directory)") - parser.add_option('-n', '--nat-address', default=None, metavar="ADDRESS", - help="the NAT address to use for passive connections") - parser.add_option('-r', '--range', default=None, metavar="FROM-TO", - help="the range of TCP ports to use for passive " - "connections (e.g. -r 8000-9000)") - parser.add_option('-D', '--debug', action='store_true', - help="enable DEBUG logging level") - parser.add_option('-v', '--version', action='store_true', - help="print pyftpdlib version and exit") - parser.add_option('-V', '--verbose', action='store_true', - help="activate a more verbose logging") - parser.add_option('-u', '--username', type=str, default=None, - help="specify username to login with (anonymous login " - "will be disabled and password required " - "if supplied)") - parser.add_option('-P', '--password', type=str, default=None, - help="specify a password to login with (username " - "required to be useful)") + parser = optparse.OptionParser( + usage=usage, + description=main.__doc__, + formatter=CustomizedOptionFormatter(), + ) + parser.add_option( + '-i', + '--interface', + default=None, + metavar="ADDRESS", + help="specify the interface to run on (default all interfaces)", + ) + parser.add_option( + '-p', + '--port', + type="int", + default=2121, + metavar="PORT", + help="specify port number to run on (default 2121)", + ) + parser.add_option( + '-w', + '--write', + action="store_true", + default=False, + help="grants write access for logged in user (default read-only)", + ) + parser.add_option( + '-d', + '--directory', + default=getcwdu(), + metavar="FOLDER", + help="specify the directory to share (default current directory)", + ) + parser.add_option( + '-n', + '--nat-address', + default=None, + metavar="ADDRESS", + help="the NAT address to use for passive connections", + ) + parser.add_option( + '-r', + '--range', + default=None, + metavar="FROM-TO", + help=( + "the range of TCP ports to use for passive " + "connections (e.g. -r 8000-9000)" + ), + ) + parser.add_option( + '-D', '--debug', action='store_true', help="enable DEBUG logging level" + ) + parser.add_option( + '-v', + '--version', + action='store_true', + help="print pyftpdlib version and exit", + ) + parser.add_option( + '-V', + '--verbose', + action='store_true', + help="activate a more verbose logging", + ) + parser.add_option( + '-u', + '--username', + type=str, + default=None, + help=( + "specify username to login with (anonymous login " + "will be disabled and password required " + "if supplied)" + ), + ) + parser.add_option( + '-P', + '--password', + type=str, + default=None, + help=( + "specify a password to login with (username required to be useful)" + ), + ) options, args = parser.parse_args() if options.version: @@ -96,11 +151,11 @@ def main(): if options.username: if not options.password: parser.error( - "if username (-u) is supplied, password ('-P') is required") - authorizer.add_user(options.username, - options.password, - options.directory, - perm=perm) + "if username (-u) is supplied, password ('-P') is required" + ) + authorizer.add_user( + options.username, options.password, options.directory, perm=perm + ) else: authorizer.add_anonymous(options.directory, perm=perm) diff --git a/pyftpdlib/_asynchat.py b/pyftpdlib/_asynchat.py index 4231570f..60b15c07 100644 --- a/pyftpdlib/_asynchat.py +++ b/pyftpdlib/_asynchat.py @@ -104,15 +104,17 @@ def handle_read(self): if index != -1: if index > 0: self.collect_incoming_data(self.ac_in_buffer[:index]) - self.ac_in_buffer = self.ac_in_buffer[index + - terminator_len:] + self.ac_in_buffer = self.ac_in_buffer[ + index + terminator_len : + ] self.found_terminator() else: index = find_prefix_at_end(self.ac_in_buffer, terminator) if index: if index != lb: self.collect_incoming_data( - self.ac_in_buffer[:-index]) + self.ac_in_buffer[:-index] + ) self.ac_in_buffer = self.ac_in_buffer[-index:] break else: @@ -127,12 +129,11 @@ def handle_close(self): def push(self, data): if not isinstance(data, (bytes, bytearray, memoryview)): - raise TypeError('data argument must be byte-ish (%r)', - type(data)) + raise TypeError('data argument must be byte-ish (%r)', type(data)) sabs = self.ac_out_buffer_size if len(data) > sabs: for i in range(0, len(data), sabs): - self.producer_fifo.append(data[i:i + sabs]) + self.producer_fifo.append(data[i : i + sabs]) else: self.producer_fifo.append(data) self.initiate_send() @@ -201,8 +202,8 @@ def __init__(self, data, buffer_size=512): def more(self): if len(self.data) > self.buffer_size: - result = self.data[:self.buffer_size] - self.data = self.data[self.buffer_size:] + result = self.data[: self.buffer_size] + self.data = self.data[self.buffer_size :] return result else: result = self.data diff --git a/pyftpdlib/_asyncore.py b/pyftpdlib/_asyncore.py index d2adfb21..23e11e27 100644 --- a/pyftpdlib/_asyncore.py +++ b/pyftpdlib/_asyncore.py @@ -47,7 +47,8 @@ _DISCONNECTED = frozenset( - {ECONNRESET, ENOTCONN, ESHUTDOWN, ECONNABORTED, EPIPE, EBADF}) + {ECONNRESET, ENOTCONN, ESHUTDOWN, ECONNABORTED, EPIPE, EBADF} +) socket_map = {} @@ -178,7 +179,7 @@ def poll2(timeout=0.0, map=None): readwrite(obj, flags) -poll3 = poll2 # Alias for backward compatibility +poll3 = poll2 # Alias for backward compatibility def loop(timeout=30.0, use_poll=False, map=None, count=None): @@ -231,8 +232,9 @@ def __init__(self, sock=None, map=None): self.socket = None def __repr__(self): - status = [self.__class__.__module__ + - "." + self.__class__.__qualname__] + status = [ + self.__class__.__module__ + "." + self.__class__.__qualname__ + ] if self.accepting and self.addr: status.append('listening') elif self.connected: @@ -272,9 +274,10 @@ def set_socket(self, sock, map=None): def set_reuse_addr(self): try: self.socket.setsockopt( - socket.SOL_SOCKET, socket.SO_REUSEADDR, - self.socket.getsockopt(socket.SOL_SOCKET, - socket.SO_REUSEADDR) | 1 + socket.SOL_SOCKET, + socket.SO_REUSEADDR, + self.socket.getsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR) + | 1, ) except OSError: pass @@ -299,8 +302,11 @@ def connect(self, address): self.connected = False self.connecting = True err = self.socket.connect_ex(address) - if err in (EINPROGRESS, EALREADY, EWOULDBLOCK) \ - or err == EINVAL and os.name == 'nt': + if ( + err in (EINPROGRESS, EALREADY, EWOULDBLOCK) + or err == EINVAL + and os.name == 'nt' + ): self.addr = address return if err in (0, EISCONN): @@ -412,13 +418,9 @@ def handle_error(self): self_repr = '<__repr__(self) failed for object at %0x>' % id(self) self.log_info( - 'uncaptured python exception, closing channel %s (%s:%s %s)' % ( - self_repr, - t, - v, - tbinfo - ), - 'error' + 'uncaptured python exception, closing channel %s (%s:%s %s)' + % (self_repr, t, v, tbinfo), + 'error', ) self.handle_close() @@ -471,6 +473,7 @@ def send(self, data): self.out_buffer = self.out_buffer + data self.initiate_send() + # --------------------------------------------------------------------------- # used for debugging. # --------------------------------------------------------------------------- @@ -485,7 +488,7 @@ def compact_traceback(): tbinfo.append(( tb.tb_frame.f_code.co_filename, tb.tb_frame.f_code.co_name, - str(tb.tb_lineno) + str(tb.tb_lineno), )) tb = tb.tb_next @@ -517,6 +520,7 @@ def close_all(map=None, ignore_all=False): if os.name == 'posix': + class file_wrapper: # Here we override just enough to make a file # look like a socket for the purposes of asyncore. @@ -527,8 +531,12 @@ def __init__(self, fd): def __del__(self): if self.fd >= 0: - warnings.warn("unclosed file %r" % self, ResourceWarning, - source=self, stacklevel=2) + warnings.warn( + "unclosed file %r" % self, + ResourceWarning, + source=self, + stacklevel=2, + ) self.close() def recv(self, *args): @@ -538,12 +546,15 @@ def send(self, *args): return os.write(self.fd, *args) def getsockopt(self, level, optname, buflen=None): - if (level == socket.SOL_SOCKET and - optname == socket.SO_ERROR and - not buflen): + if ( + level == socket.SOL_SOCKET + and optname == socket.SO_ERROR + and not buflen + ): return 0 - raise NotImplementedError("Only asyncore specific behaviour " - "implemented.") + raise NotImplementedError( + "Only asyncore specific behaviour implemented." + ) read = recv write = send diff --git a/pyftpdlib/_compat.py b/pyftpdlib/_compat.py index e4db3f61..3c86fd27 100644 --- a/pyftpdlib/_compat.py +++ b/pyftpdlib/_compat.py @@ -17,6 +17,7 @@ _SENTINEL = object() if PY3: + def u(s): return s @@ -28,6 +29,7 @@ def b(s): xrange = range long = int else: + def u(s): return unicode(s) @@ -44,6 +46,7 @@ def b(s): try: callable = callable except Exception: + def callable(obj): return any("__call__" in klass.__dict__ for klass in type(obj).__mro__) @@ -70,7 +73,9 @@ def __init__(self, *args, **kwargs): if not attr.startswith('__'): setattr(self, attr, getattr(unwrap_me, attr)) else: - super(TemporaryClass, self).__init__(*args, **kwargs) # noqa + super(TemporaryClass, self).__init__( # noqa + *args, **kwargs + ) class __metaclass__(type): def __instancecheck__(cls, inst): @@ -102,7 +107,8 @@ def FileExistsError(inst): except OSError: raise RuntimeError( "broken or incompatible Python implementation, see: " - "https://github.com/giampaolo/psutil/issues/1659") + "https://github.com/giampaolo/psutil/issues/1659" + ) # Python 3 super(). diff --git a/pyftpdlib/authorizers.py b/pyftpdlib/authorizers.py index 6e6553d0..5bd1b62f 100644 --- a/pyftpdlib/authorizers.py +++ b/pyftpdlib/authorizers.py @@ -27,16 +27,18 @@ class for: from ._compat import unicode -__all__ = ['DummyAuthorizer', - # 'BaseUnixAuthorizer', 'UnixAuthorizer', - # 'BaseWindowsAuthorizer', 'WindowsAuthorizer', - ] +__all__ = [ + 'DummyAuthorizer', + # 'BaseUnixAuthorizer', 'UnixAuthorizer', + # 'BaseWindowsAuthorizer', 'WindowsAuthorizer', +] # =================================================================== # --- exceptions # =================================================================== + class AuthorizerError(Exception): """Base class for authorizer exceptions.""" @@ -49,6 +51,7 @@ class AuthenticationFailed(Exception): # --- base class # =================================================================== + class DummyAuthorizer: """Basic "dummy" authorizer class, suitable for subclassing to create your own custom authorizers. @@ -71,8 +74,15 @@ class and overriding appropriate methods as necessary. def __init__(self): self.user_table = {} - def add_user(self, username, password, homedir, perm='elr', - msg_login="Login successful.", msg_quit="Goodbye."): + def add_user( + self, + username, + password, + homedir, + perm='elr', + msg_login="Login successful.", + msg_quit="Goodbye.", + ): """Add a user to the virtual users table. AuthorizerError exceptions raised on error conditions such as @@ -106,13 +116,14 @@ def add_user(self, username, password, homedir, perm='elr', raise ValueError('no such directory: %r' % homedir) homedir = os.path.realpath(homedir) self._check_permissions(username, perm) - dic = {'pwd': str(password), - 'home': homedir, - 'perm': perm, - 'operms': {}, - 'msg_login': str(msg_login), - 'msg_quit': str(msg_quit) - } + dic = { + 'pwd': str(password), + 'home': homedir, + 'perm': perm, + 'operms': {}, + 'msg_login': str(msg_login), + 'msg_quit': str(msg_quit), + } self.user_table[username] = dic def add_anonymous(self, homedir, **kwargs): @@ -210,8 +221,11 @@ def has_perm(self, username, perm, path=None): if self._issubpath(path, dir): if recursive: return perm in operm - if (path == dir or os.path.dirname(path) == dir and not - os.path.isdir(path)): + if ( + path == dir + or os.path.dirname(path) == dir + and not os.path.isdir(path) + ): return perm in operm return perm in self.user_table[username]['perm'] @@ -236,18 +250,23 @@ def _check_permissions(self, username, perm): for p in perm: if p not in self.read_perms + self.write_perms: raise ValueError('no such permission %r' % p) - if username == 'anonymous' and \ - p in self.write_perms and not \ - warned: - warnings.warn("write permissions assigned to anonymous user.", - RuntimeWarning, stacklevel=2) + if ( + username == 'anonymous' + and p in self.write_perms + and not warned + ): + warnings.warn( + "write permissions assigned to anonymous user.", + RuntimeWarning, + stacklevel=2, + ) warned = 1 def _issubpath(self, a, b): """Return True if a is a sub-path of b or if the paths are equal.""" p1 = a.rstrip(os.sep).split(os.sep) p2 = b.rstrip(os.sep).split(os.sep) - return p1[:len(p2)] == p2 + return p1[: len(p2)] == p2 def replace_anonymous(callable): @@ -260,6 +279,7 @@ def wrapper(self, username, *args, **kwargs): if username == 'anonymous': username = self.anonymous_user or username return callable(self, username, *args, **kwargs) + return wrapper @@ -267,6 +287,7 @@ def wrapper(self, username, *args, **kwargs): # --- platform specific authorizers # =================================================================== + class _Base: """Methods common to both Unix and Windows authorizers. Not supposed to be used directly. @@ -281,11 +302,13 @@ class _Base: def __init__(self): """Check for errors in the constructor.""" if self.rejected_users and self.allowed_users: - raise AuthorizerError("rejected_users and allowed_users options " - "are mutually exclusive") + raise AuthorizerError( + "rejected_users and allowed_users options " + "are mutually exclusive" + ) users = self._get_system_users() - for user in (self.allowed_users or self.rejected_users): + for user in self.allowed_users or self.rejected_users: if user == 'anonymous': raise AuthorizerError('invalid username "anonymous"') if user not in users: @@ -296,18 +319,32 @@ def __init__(self): raise AuthorizerError('no such user %s' % self.anonymous_user) home = self.get_home_dir(self.anonymous_user) if not os.path.isdir(home): - raise AuthorizerError('no valid home set for user %s' - % self.anonymous_user) - - def override_user(self, username, password=None, homedir=None, perm=None, - msg_login=None, msg_quit=None): + raise AuthorizerError( + 'no valid home set for user %s' % self.anonymous_user + ) + + def override_user( + self, + username, + password=None, + homedir=None, + perm=None, + msg_login=None, + msg_quit=None, + ): """Overrides the options specified in the class constructor for a specific user. """ - if (not password and not homedir and not perm and not msg_login and not - msg_quit): + if ( + not password + and not homedir + and not perm + and not msg_login + and not msg_quit + ): raise AuthorizerError( - "at least one keyword argument must be specified") + "at least one keyword argument must be specified" + ) if self.allowed_users and username not in self.allowed_users: raise AuthorizerError('%s is not an allowed user' % username) if self.rejected_users and username in self.rejected_users: @@ -322,12 +359,14 @@ def override_user(self, username, password=None, homedir=None, perm=None, if username in self._dummy_authorizer.user_table: # re-set parameters del self._dummy_authorizer.user_table[username] - self._dummy_authorizer.add_user(username, - password or "", - homedir or getcwdu(), - perm or "", - msg_login or "", - msg_quit or "") + self._dummy_authorizer.add_user( + username, + password or "", + homedir or getcwdu(), + perm or "", + msg_login or "", + msg_quit or "", + ) if homedir is None: self._dummy_authorizer.user_table[username]['home'] = "" @@ -497,47 +536,50 @@ class UnixAuthorizer(_Base, BaseUnixAuthorizer): # --- public API - def __init__(self, global_perm="elradfmwMT", - allowed_users=None, - rejected_users=None, - require_valid_shell=True, - anonymous_user=None, - msg_login="Login successful.", - msg_quit="Goodbye."): + def __init__( + self, + global_perm="elradfmwMT", + allowed_users=None, + rejected_users=None, + require_valid_shell=True, + anonymous_user=None, + msg_login="Login successful.", + msg_quit="Goodbye.", + ): """Parameters: - - (string) global_perm: - a series of letters referencing the users permissions; - defaults to "elradfmwMT" which means full read and write - access for everybody (except anonymous). - - - (list) allowed_users: - a list of users which are accepted for authenticating - against the FTP server; defaults to [] (no restrictions). - - - (list) rejected_users: - a list of users which are not accepted for authenticating - against the FTP server; defaults to [] (no restrictions). - - - (bool) require_valid_shell: - Deny access for those users which do not have a valid shell - binary listed in /etc/shells. - If /etc/shells cannot be found this is a no-op. - Anonymous user is not subject to this option, and is free - to not have a valid shell defined. - Defaults to True (a valid shell is required for login). - - - (string) anonymous_user: - specify it if you intend to provide anonymous access. - The value expected is a string representing the system user - to use for managing anonymous sessions; defaults to None - (anonymous access disabled). - - - (string) msg_login: - the string sent when client logs in. - - - (string) msg_quit: - the string sent when client quits. + - (string) global_perm: + a series of letters referencing the users permissions; + defaults to "elradfmwMT" which means full read and write + access for everybody (except anonymous). + + - (list) allowed_users: + a list of users which are accepted for authenticating + against the FTP server; defaults to [] (no restrictions). + + - (list) rejected_users: + a list of users which are not accepted for authenticating + against the FTP server; defaults to [] (no restrictions). + + - (bool) require_valid_shell: + Deny access for those users which do not have a valid shell + binary listed in /etc/shells. + If /etc/shells cannot be found this is a no-op. + Anonymous user is not subject to this option, and is free + to not have a valid shell defined. + Defaults to True (a valid shell is required for login). + + - (string) anonymous_user: + specify it if you intend to provide anonymous access. + The value expected is a string representing the system user + to use for managing anonymous sessions; defaults to None + (anonymous access disabled). + + - (string) msg_login: + the string sent when client logs in. + + - (string) msg_quit: + the string sent when client quits. """ BaseUnixAuthorizer.__init__(self, anonymous_user) if allowed_users is None: @@ -558,19 +600,28 @@ def __init__(self, global_perm="elradfmwMT", if require_valid_shell: for username in self.allowed_users: if not self._has_valid_shell(username): - raise AuthorizerError("user %s has not a valid shell" - % username) - - def override_user(self, username, password=None, homedir=None, - perm=None, msg_login=None, msg_quit=None): + raise AuthorizerError( + "user %s has not a valid shell" % username + ) + + def override_user( + self, + username, + password=None, + homedir=None, + perm=None, + msg_login=None, + msg_quit=None, + ): """Overrides the options specified in the class constructor for a specific user. """ if self.require_valid_shell and username != 'anonymous': if not self._has_valid_shell(username): raise AuthorizerError(self.msg_invalid_shell % username) - _Base.override_user(self, username, password, homedir, perm, - msg_login, msg_quit) + _Base.override_user( + self, username, password, homedir, perm, msg_login, msg_quit + ) # --- overridden / private API @@ -586,12 +637,14 @@ def validate_authentication(self, username, password, handler): if overridden_password != password: raise AuthenticationFailed(self.msg_wrong_password) else: - BaseUnixAuthorizer.validate_authentication(self, username, - password, handler) + BaseUnixAuthorizer.validate_authentication( + self, username, password, handler + ) if self.require_valid_shell and username != 'anonymous': if not self._has_valid_shell(username): raise AuthenticationFailed( - self.msg_invalid_shell % username) + self.msg_invalid_shell % username + ) @replace_anonymous def has_user(self, username): @@ -665,8 +718,9 @@ def __init__(self, anonymous_user=None, anonymous_password=None): self.anonymous_user = anonymous_user self.anonymous_password = anonymous_password if self.anonymous_user is not None: - self.impersonate_user(self.anonymous_user, - self.anonymous_password) + self.impersonate_user( + self.anonymous_user, self.anonymous_password + ) self.terminate_impersonation(None) def validate_authentication(self, username, password, handler): @@ -675,9 +729,13 @@ def validate_authentication(self, username, password, handler): raise AuthenticationFailed(self.msg_anon_not_allowed) return try: - win32security.LogonUser(username, None, password, - win32con.LOGON32_LOGON_INTERACTIVE, - win32con.LOGON32_PROVIDER_DEFAULT) + win32security.LogonUser( + username, + None, + password, + win32con.LOGON32_LOGON_INTERACTIVE, + win32con.LOGON32_PROVIDER_DEFAULT, + ) except pywintypes.error: raise AuthenticationFailed(self.msg_wrong_password) @@ -685,9 +743,12 @@ def validate_authentication(self, username, password, handler): def impersonate_user(self, username, password): """Impersonate the security context of another user.""" handler = win32security.LogonUser( - username, None, password, + username, + None, + password, win32con.LOGON32_LOGON_INTERACTIVE, - win32con.LOGON32_PROVIDER_DEFAULT) + win32con.LOGON32_PROVIDER_DEFAULT, + ) win32security.ImpersonateLoggedOnUser(handler) handler.Close() @@ -706,7 +767,8 @@ def get_home_dir(self, username): """ try: sid = win32security.ConvertSidToStringSid( - win32security.LookupAccountName(None, username)[0]) + win32security.LookupAccountName(None, username)[0] + ) except pywintypes.error as err: raise AuthorizerError(err) path = r"SOFTWARE\Microsoft\Windows NT" @@ -715,7 +777,8 @@ def get_home_dir(self, username): key = winreg.OpenKey(winreg.HKEY_LOCAL_MACHINE, path) except WindowsError: raise AuthorizerError( - "No profile directory defined for user %s" % username) + "No profile directory defined for user %s" % username + ) value = winreg.QueryValueEx(key, "ProfileImagePath")[0] home = win32api.ExpandEnvironmentStrings(value) if not PY3 and not isinstance(home, unicode): @@ -727,8 +790,9 @@ def _get_system_users(cls): """Return all users defined on the Windows system.""" # XXX - Does Windows allow usernames with chars outside of # ASCII set? In that case we need to convert this to unicode. - return [entry['name'] for entry in - win32net.NetUserEnum(None, 0)[0]] + return [ + entry['name'] for entry in win32net.NetUserEnum(None, 0)[0] + ] def get_msg_login(self, username): return "Login successful." @@ -762,47 +826,49 @@ class WindowsAuthorizer(_Base, BaseWindowsAuthorizer): # --- public API - def __init__(self, - global_perm="elradfmwMT", - allowed_users=None, - rejected_users=None, - anonymous_user=None, - anonymous_password=None, - msg_login="Login successful.", - msg_quit="Goodbye."): + def __init__( + self, + global_perm="elradfmwMT", + allowed_users=None, + rejected_users=None, + anonymous_user=None, + anonymous_password=None, + msg_login="Login successful.", + msg_quit="Goodbye.", + ): """Parameters: - - (string) global_perm: - a series of letters referencing the users permissions; - defaults to "elradfmwMT" which means full read and write - access for everybody (except anonymous). - - - (list) allowed_users: - a list of users which are accepted for authenticating - against the FTP server; defaults to [] (no restrictions). - - - (list) rejected_users: - a list of users which are not accepted for authenticating - against the FTP server; defaults to [] (no restrictions). - - - (string) anonymous_user: - specify it if you intend to provide anonymous access. - The value expected is a string representing the system user - to use for managing anonymous sessions. - As for IIS, it is recommended to use Guest account. - The common practice is to first enable the Guest user, which - is disabled by default and then assign an empty password. - Defaults to None (anonymous access disabled). - - - (string) anonymous_password: - the password of the user who has been chosen to manage the - anonymous sessions. Defaults to None (empty password). - - - (string) msg_login: - the string sent when client logs in. - - - (string) msg_quit: - the string sent when client quits. + - (string) global_perm: + a series of letters referencing the users permissions; + defaults to "elradfmwMT" which means full read and write + access for everybody (except anonymous). + + - (list) allowed_users: + a list of users which are accepted for authenticating + against the FTP server; defaults to [] (no restrictions). + + - (list) rejected_users: + a list of users which are not accepted for authenticating + against the FTP server; defaults to [] (no restrictions). + + - (string) anonymous_user: + specify it if you intend to provide anonymous access. + The value expected is a string representing the system user + to use for managing anonymous sessions. + As for IIS, it is recommended to use Guest account. + The common practice is to first enable the Guest user, which + is disabled by default and then assign an empty password. + Defaults to None (anonymous access disabled). + + - (string) anonymous_password: + the password of the user who has been chosen to manage the + anonymous sessions. Defaults to None (empty password). + + - (string) msg_login: + the string sent when client logs in. + + - (string) msg_quit: + the string sent when client quits. """ if allowed_users is None: allowed_users = [] @@ -820,17 +886,26 @@ def __init__(self, _Base.__init__(self) # actually try to impersonate the user if self.anonymous_user is not None: - self.impersonate_user(self.anonymous_user, - self.anonymous_password) + self.impersonate_user( + self.anonymous_user, self.anonymous_password + ) self.terminate_impersonation(None) - def override_user(self, username, password=None, homedir=None, - perm=None, msg_login=None, msg_quit=None): + def override_user( + self, + username, + password=None, + homedir=None, + perm=None, + msg_login=None, + msg_quit=None, + ): """Overrides the options specified in the class constructor for a specific user. """ - _Base.override_user(self, username, password, homedir, perm, - msg_login, msg_quit) + _Base.override_user( + self, username, password, homedir, perm, msg_login, msg_quit + ) # --- overridden / private API @@ -853,7 +928,8 @@ def validate_authentication(self, username, password, handler): raise AuthenticationFailed(self.msg_wrong_password) else: BaseWindowsAuthorizer.validate_authentication( - self, username, password, handler) + self, username, password, handler + ) def impersonate_user(self, username, password): """Impersonate the security context of another user.""" diff --git a/pyftpdlib/filesystems.py b/pyftpdlib/filesystems.py index ff74ffa9..d2b9fd74 100644 --- a/pyftpdlib/filesystems.py +++ b/pyftpdlib/filesystems.py @@ -26,14 +26,27 @@ __all__ = ['FilesystemError', 'AbstractedFS'] -_months_map = {1: 'Jan', 2: 'Feb', 3: 'Mar', 4: 'Apr', 5: 'May', 6: 'Jun', - 7: 'Jul', 8: 'Aug', 9: 'Sep', 10: 'Oct', 11: 'Nov', 12: 'Dec'} +_months_map = { + 1: 'Jan', + 2: 'Feb', + 3: 'Mar', + 4: 'Apr', + 5: 'May', + 6: 'Jun', + 7: 'Jul', + 8: 'Aug', + 9: 'Sep', + 10: 'Oct', + 11: 'Nov', + 12: 'Dec', +} def _memoize(fun): """A simple memoize decorator for functions supporting (hashable) positional arguments. """ + def wrapper(*args, **kwargs): key = (args, frozenset(sorted(kwargs.items()))) try: @@ -50,6 +63,7 @@ def wrapper(*args, **kwargs): # --- custom exceptions # =================================================================== + class FilesystemError(Exception): """Custom class for filesystem-related exceptions. You can raise this from an AbstractedFS subclass in order to @@ -61,6 +75,7 @@ class FilesystemError(Exception): # --- base class # =================================================================== + class AbstractedFS: """A class used to interact with the file system, providing a cross-platform interface compatible with both Windows and @@ -82,8 +97,8 @@ class AbstractedFS: def __init__(self, root, cmd_channel): """ - - (str) root: the user "real" home directory (e.g. '/home/user') - - (instance) cmd_channel: the FTPHandler class instance. + - (str) root: the user "real" home directory (e.g. '/home/user') + - (instance) cmd_channel: the FTPHandler class instance. """ assert isinstance(root, unicode) # Set initial current working directory. @@ -191,7 +206,7 @@ def fs2ftp(self, fspath): if not self.validpath(p): return u('/') p = p.replace(os.sep, "/") - p = p[len(self.root):] + p = p[len(self.root) :] if not p.startswith('/'): p = '/' + p return p @@ -213,7 +228,7 @@ def validpath(self, path): root = root + os.sep if not path.endswith(os.sep): path = path + os.sep - if path[0:len(root)] == root: + if path[0 : len(root)] == root: return True return False @@ -229,6 +244,7 @@ def mkstemp(self, suffix='', prefix='', dir=None, mode='wb'): name. Unlike mkstemp it returns an object with a file-like interface. """ + class FileWrapper: def __init__(self, fd, name): @@ -307,15 +323,18 @@ def utime(self, path, timeval): return os.utime(path, (timeval, timeval)) if hasattr(os, 'lstat'): + def lstat(self, path): """Like stat but does not follow symbolic links.""" # on python 2 we might also get bytes from os.lisdir() # assert isinstance(path, unicode), path return os.lstat(path) + else: lstat = stat if hasattr(os, 'readlink'): + def readlink(self, path): """Return a string representing the path to which a symbolic link points. @@ -367,6 +386,7 @@ def lexists(self, path): return os.path.lexists(path) if pwd is not None: + def get_user_by_uid(self, uid): """Return the username associated with user id. If this can't be determined return raw uid instead. @@ -376,11 +396,14 @@ def get_user_by_uid(self, uid): return pwd.getpwuid(uid).pw_name except KeyError: return uid + else: + def get_user_by_uid(self, uid): return "owner" if grp is not None: + def get_group_by_gid(self, gid): """Return the group name associated with group id. If this can't be determined return raw gid instead. @@ -390,7 +413,9 @@ def get_group_by_gid(self, gid): return grp.getgrgid(gid).gr_name except KeyError: return gid + else: + def get_group_by_gid(self, gid): return "group" @@ -417,6 +442,7 @@ def format_list(self, basedir, listing, ignore_err=True): drwxrwxrwx 1 owner group 0 Aug 31 18:50 e-books -rw-rw-rw- 1 owner group 380 Sep 02 3:40 module.py """ + @_memoize def get_user_by_uid(uid): return self.get_user_by_uid(uid) @@ -468,15 +494,19 @@ def get_group_by_gid(gid): # https://github.com/giampaolo/pyftpdlib/issues/187 fmtstr = '%d %Y' if now - st.st_mtime > SIX_MONTHS else '%d %H:%M' try: - mtimestr = "%s %s" % (_months_map[mtime.tm_mon], - time.strftime(fmtstr, mtime)) + mtimestr = "%s %s" % ( + _months_map[mtime.tm_mon], + time.strftime(fmtstr, mtime), + ) except ValueError: # It could be raised if last mtime happens to be too # old (prior to year 1900) in which case we return # the current time as last mtime. mtime = timefunc() - mtimestr = "%s %s" % (_months_map[mtime.tm_mon], - time.strftime("%d %H:%M", mtime)) + mtimestr = "%s %s" % ( + _months_map[mtime.tm_mon], + time.strftime("%d %H:%M", mtime), + ) # same as stat.S_ISLNK(st.st_mode) but slighlty faster islink = (st.st_mode & 61440) == stat.S_IFLNK @@ -491,7 +521,14 @@ def get_group_by_gid(gid): # formatting is matched with proftpd ls output line = "%s %3s %-8s %-8s %8s %s %s\r\n" % ( - perms, nlinks, uname, gname, size, mtimestr, basename) + perms, + nlinks, + uname, + gname, + size, + mtimestr, + basename, + ) yield line.encode('utf8', self.cmd_channel.unicode_errors) def format_mlsx(self, basedir, listing, perms, facts, ignore_err=True): @@ -588,8 +625,9 @@ def format_mlsx(self, basedir, listing, perms, facts, ignore_err=True): # last modification time if show_modify: try: - retfacts['modify'] = time.strftime("%Y%m%d%H%M%S", - timefunc(st.st_mtime)) + retfacts['modify'] = time.strftime( + "%Y%m%d%H%M%S", timefunc(st.st_mtime) + ) # it could be raised if last mtime happens to be too old # (prior to year 1900) except ValueError: @@ -597,8 +635,9 @@ def format_mlsx(self, basedir, listing, perms, facts, ignore_err=True): if show_create: # on Windows we can provide also the creation time try: - retfacts['create'] = time.strftime("%Y%m%d%H%M%S", - timefunc(st.st_ctime)) + retfacts['create'] = time.strftime( + "%Y%m%d%H%M%S", timefunc(st.st_ctime) + ) except ValueError: pass # UNIX only @@ -621,8 +660,9 @@ def format_mlsx(self, basedir, listing, perms, facts, ignore_err=True): retfacts['unique'] = "%xg%x" % (st.st_dev, st.st_ino) # facts can be in any order but we sort them by name - factstring = "".join(["%s=%s;" % (x, retfacts[x]) - for x in sorted(retfacts.keys())]) + factstring = "".join( + ["%s=%s;" % (x, retfacts[x]) for x in sorted(retfacts.keys())] + ) line = "%s %s\r\n" % (factstring, basename) yield line.encode('utf8', self.cmd_channel.unicode_errors) diff --git a/pyftpdlib/handlers.py b/pyftpdlib/handlers.py index dac08936..0bcbdf74 100644 --- a/pyftpdlib/handlers.py +++ b/pyftpdlib/handlers.py @@ -90,147 +90,284 @@ def _import_sendfile(): proto_cmds = { 'ABOR': dict( - perm=None, auth=True, arg=False, - help='Syntax: ABOR (abort transfer).'), + perm=None, auth=True, arg=False, help='Syntax: ABOR (abort transfer).' + ), 'ALLO': dict( - perm=None, auth=True, arg=True, - help='Syntax: ALLO bytes (noop; allocate storage).'), + perm=None, + auth=True, + arg=True, + help='Syntax: ALLO bytes (noop; allocate storage).', + ), 'APPE': dict( - perm='a', auth=True, arg=True, - help='Syntax: APPE file-name (append data to file).'), + perm='a', + auth=True, + arg=True, + help='Syntax: APPE file-name (append data to file).', + ), 'CDUP': dict( - perm='e', auth=True, arg=False, - help='Syntax: CDUP (go to parent directory).'), + perm='e', + auth=True, + arg=False, + help='Syntax: CDUP (go to parent directory).', + ), 'CWD': dict( - perm='e', auth=True, arg=None, - help='Syntax: CWD [ dir-name] (change working directory).'), + perm='e', + auth=True, + arg=None, + help='Syntax: CWD [ dir-name] (change working directory).', + ), 'DELE': dict( - perm='d', auth=True, arg=True, - help='Syntax: DELE file-name (delete file).'), + perm='d', + auth=True, + arg=True, + help='Syntax: DELE file-name (delete file).', + ), 'EPRT': dict( - perm=None, auth=True, arg=True, - help='Syntax: EPRT |proto|ip|port| (extended active mode).'), + perm=None, + auth=True, + arg=True, + help='Syntax: EPRT |proto|ip|port| (extended active mode).', + ), 'EPSV': dict( - perm=None, auth=True, arg=None, - help='Syntax: EPSV [ proto/"ALL"] (extended passive mode).'), + perm=None, + auth=True, + arg=None, + help='Syntax: EPSV [ proto/"ALL"] (extended passive mode).', + ), 'FEAT': dict( - perm=None, auth=False, arg=False, - help='Syntax: FEAT (list all new features supported).'), + perm=None, + auth=False, + arg=False, + help='Syntax: FEAT (list all new features supported).', + ), 'HELP': dict( - perm=None, auth=False, arg=None, - help='Syntax: HELP [ cmd] (show help).'), + perm=None, + auth=False, + arg=None, + help='Syntax: HELP [ cmd] (show help).', + ), 'LIST': dict( - perm='l', auth=True, arg=None, - help='Syntax: LIST [ path] (list files).'), + perm='l', + auth=True, + arg=None, + help='Syntax: LIST [ path] (list files).', + ), 'MDTM': dict( - perm='l', auth=True, arg=True, - help='Syntax: MDTM [ path] (file last modification time).'), + perm='l', + auth=True, + arg=True, + help='Syntax: MDTM [ path] (file last modification time).', + ), 'MFMT': dict( - perm='T', auth=True, arg=True, - help='Syntax: MFMT timeval path (file update last ' - 'modification time).'), + perm='T', + auth=True, + arg=True, + help=( + 'Syntax: MFMT timeval path (file update last ' + 'modification time).' + ), + ), 'MLSD': dict( - perm='l', auth=True, arg=None, - help='Syntax: MLSD [ path] (list directory).'), + perm='l', + auth=True, + arg=None, + help='Syntax: MLSD [ path] (list directory).', + ), 'MLST': dict( - perm='l', auth=True, arg=None, - help='Syntax: MLST [ path] (show information about path).'), + perm='l', + auth=True, + arg=None, + help='Syntax: MLST [ path] (show information about path).', + ), 'MODE': dict( - perm=None, auth=True, arg=True, - help='Syntax: MODE mode (noop; set data transfer mode).'), + perm=None, + auth=True, + arg=True, + help='Syntax: MODE mode (noop; set data transfer mode).', + ), 'MKD': dict( - perm='m', auth=True, arg=True, - help='Syntax: MKD path (create directory).'), + perm='m', + auth=True, + arg=True, + help='Syntax: MKD path (create directory).', + ), 'NLST': dict( - perm='l', auth=True, arg=None, - help='Syntax: NLST [ path] (list path in a compact form).'), + perm='l', + auth=True, + arg=None, + help='Syntax: NLST [ path] (list path in a compact form).', + ), 'NOOP': dict( - perm=None, auth=False, arg=False, - help='Syntax: NOOP (just do nothing).'), + perm=None, + auth=False, + arg=False, + help='Syntax: NOOP (just do nothing).', + ), 'OPTS': dict( - perm=None, auth=True, arg=True, - help='Syntax: OPTS cmd [ option] (set option for command).'), + perm=None, + auth=True, + arg=True, + help='Syntax: OPTS cmd [ option] (set option for command).', + ), 'PASS': dict( - perm=None, auth=False, arg=None, - help='Syntax: PASS [ password] (set user password).'), + perm=None, + auth=False, + arg=None, + help='Syntax: PASS [ password] (set user password).', + ), 'PASV': dict( - perm=None, auth=True, arg=False, - help='Syntax: PASV (open passive data connection).'), + perm=None, + auth=True, + arg=False, + help='Syntax: PASV (open passive data connection).', + ), 'PORT': dict( - perm=None, auth=True, arg=True, - help='Syntax: PORT h,h,h,h,p,p (open active data connection).'), + perm=None, + auth=True, + arg=True, + help='Syntax: PORT h,h,h,h,p,p (open active data connection).', + ), 'PWD': dict( - perm=None, auth=True, arg=False, - help='Syntax: PWD (get current working directory).'), + perm=None, + auth=True, + arg=False, + help='Syntax: PWD (get current working directory).', + ), 'QUIT': dict( - perm=None, auth=False, arg=False, - help='Syntax: QUIT (quit current session).'), + perm=None, + auth=False, + arg=False, + help='Syntax: QUIT (quit current session).', + ), 'REIN': dict( - perm=None, auth=True, arg=False, - help='Syntax: REIN (flush account).'), + perm=None, auth=True, arg=False, help='Syntax: REIN (flush account).' + ), 'REST': dict( - perm=None, auth=True, arg=True, - help='Syntax: REST offset (set file offset).'), + perm=None, + auth=True, + arg=True, + help='Syntax: REST offset (set file offset).', + ), 'RETR': dict( - perm='r', auth=True, arg=True, - help='Syntax: RETR file-name (retrieve a file).'), + perm='r', + auth=True, + arg=True, + help='Syntax: RETR file-name (retrieve a file).', + ), 'RMD': dict( - perm='d', auth=True, arg=True, - help='Syntax: RMD dir-name (remove directory).'), + perm='d', + auth=True, + arg=True, + help='Syntax: RMD dir-name (remove directory).', + ), 'RNFR': dict( - perm='f', auth=True, arg=True, - help='Syntax: RNFR file-name (rename (source name)).'), + perm='f', + auth=True, + arg=True, + help='Syntax: RNFR file-name (rename (source name)).', + ), 'RNTO': dict( - perm='f', auth=True, arg=True, - help='Syntax: RNTO file-name (rename (destination name)).'), + perm='f', + auth=True, + arg=True, + help='Syntax: RNTO file-name (rename (destination name)).', + ), 'SITE': dict( - perm=None, auth=False, arg=True, - help='Syntax: SITE site-command (execute SITE command).'), + perm=None, + auth=False, + arg=True, + help='Syntax: SITE site-command (execute SITE command).', + ), 'SITE HELP': dict( - perm=None, auth=False, arg=None, - help='Syntax: SITE HELP [ cmd] (show SITE command help).'), + perm=None, + auth=False, + arg=None, + help='Syntax: SITE HELP [ cmd] (show SITE command help).', + ), 'SITE CHMOD': dict( - perm='M', auth=True, arg=True, - help='Syntax: SITE CHMOD mode path (change file mode).'), + perm='M', + auth=True, + arg=True, + help='Syntax: SITE CHMOD mode path (change file mode).', + ), 'SIZE': dict( - perm='l', auth=True, arg=True, - help='Syntax: SIZE file-name (get file size).'), + perm='l', + auth=True, + arg=True, + help='Syntax: SIZE file-name (get file size).', + ), 'STAT': dict( - perm='l', auth=False, arg=None, - help='Syntax: STAT [ path name] (server stats [list files]).'), + perm='l', + auth=False, + arg=None, + help='Syntax: STAT [ path name] (server stats [list files]).', + ), 'STOR': dict( - perm='w', auth=True, arg=True, - help='Syntax: STOR file-name (store a file).'), + perm='w', + auth=True, + arg=True, + help='Syntax: STOR file-name (store a file).', + ), 'STOU': dict( - perm='w', auth=True, arg=None, - help='Syntax: STOU [ name] (store a file with a unique name).'), + perm='w', + auth=True, + arg=None, + help='Syntax: STOU [ name] (store a file with a unique name).', + ), 'STRU': dict( - perm=None, auth=True, arg=True, - help='Syntax: STRU type (noop; set file structure).'), + perm=None, + auth=True, + arg=True, + help='Syntax: STRU type (noop; set file structure).', + ), 'SYST': dict( - perm=None, auth=False, arg=False, - help='Syntax: SYST (get operating system type).'), + perm=None, + auth=False, + arg=False, + help='Syntax: SYST (get operating system type).', + ), 'TYPE': dict( - perm=None, auth=True, arg=True, - help='Syntax: TYPE [A | I] (set transfer type).'), + perm=None, + auth=True, + arg=True, + help='Syntax: TYPE [A | I] (set transfer type).', + ), 'USER': dict( - perm=None, auth=False, arg=True, - help='Syntax: USER user-name (set username).'), + perm=None, + auth=False, + arg=True, + help='Syntax: USER user-name (set username).', + ), 'XCUP': dict( - perm='e', auth=True, arg=False, - help='Syntax: XCUP (obsolete; go to parent directory).'), + perm='e', + auth=True, + arg=False, + help='Syntax: XCUP (obsolete; go to parent directory).', + ), 'XCWD': dict( - perm='e', auth=True, arg=None, - help='Syntax: XCWD [ dir-name] (obsolete; change directory).'), + perm='e', + auth=True, + arg=None, + help='Syntax: XCWD [ dir-name] (obsolete; change directory).', + ), 'XMKD': dict( - perm='m', auth=True, arg=True, - help='Syntax: XMKD dir-name (obsolete; create directory).'), + perm='m', + auth=True, + arg=True, + help='Syntax: XMKD dir-name (obsolete; create directory).', + ), 'XPWD': dict( - perm=None, auth=True, arg=False, - help='Syntax: XPWD (obsolete; get current dir).'), + perm=None, + auth=True, + arg=False, + help='Syntax: XPWD (obsolete; get current dir).', + ), 'XRMD': dict( - perm='d', auth=True, arg=True, - help='Syntax: XRMD dir-name (obsolete; remove directory).'), + perm='d', + auth=True, + arg=True, + help='Syntax: XRMD dir-name (obsolete; remove directory).', + ), } if not hasattr(os, 'chmod'): @@ -284,6 +421,7 @@ class _GiveUpOnSendfile(Exception): # --- DTP classes + class PassiveDTP(Acceptor): """Creates a socket listening on a local port, dispatching the resultant connection to DTPHandler. Used for handling PASV command. @@ -295,14 +433,15 @@ class PassiveDTP(Acceptor): to listen(). If a connection request arrives when the queue is full the client may raise ECONNRESET. Defaults to 5. """ + timeout = 30 backlog = None def __init__(self, cmd_channel, extmode=False): """Initialize the passive data server. - - (instance) cmd_channel: the command channel class instance. - - (bool) extmode: whether use extended passive mode response type. + - (instance) cmd_channel: the command channel class instance. + - (bool) extmode: whether use extended passive mode response type. """ self.cmd_channel = cmd_channel self.log = cmd_channel.log @@ -353,7 +492,7 @@ def __init__(self, cmd_channel, extmode=False): "Can't find a valid passive port in the " "configured range. A random kernel-assigned " "port will be used.", - logfun=logger.warning + logfun=logger.warning, ) else: raise @@ -377,11 +516,15 @@ def __init__(self, cmd_channel, extmode=False): # The format of 227 response in not standardized. # This is the most expected: resp = '227 Entering passive mode (%s,%d,%d).' % ( - ip.replace('.', ','), port // 256, port % 256) + ip.replace('.', ','), + port // 256, + port % 256, + ) self.cmd_channel.respond(resp) else: - self.cmd_channel.respond('229 Entering extended passive mode ' - '(|||%d|).' % port) + self.cmd_channel.respond( + '229 Entering extended passive mode (|||%d|).' % port + ) if self.timeout: self.call_later(self.timeout, self.handle_timeout) @@ -401,15 +544,19 @@ def handle_accepted(self, sock, addr): sock.close() except socket.error: pass - msg = '425 Rejected data connection from foreign address ' \ + msg = ( + '425 Rejected data connection from foreign address ' + '%s:%s.' % (addr[0], addr[1]) + ) self.cmd_channel.respond_w_warning(msg) # do not close listening socket: it couldn't be client's blame return else: # site-to-site FTP allowed - msg = 'Established data connection with foreign address ' \ + msg = ( + 'Established data connection with foreign address ' + '%s:%s.' % (addr[0], addr[1]) + ) self.cmd_channel.log(msg, logfun=logger.warning) # Immediately close the current channel (we accept only one # connection at time) and avoid running out of max connections @@ -424,8 +571,9 @@ def handle_accepted(self, sock, addr): def handle_timeout(self): if self.cmd_channel.connected: - self.cmd_channel.respond("421 Passive data channel timed out.", - logfun=logger.info) + self.cmd_channel.respond( + "421 Passive data channel timed out.", logfun=logger.info + ) self.close() def handle_error(self): @@ -451,6 +599,7 @@ class ActiveDTP(Connector): - (int) timeout: the timeout for us to establish connection with the client's listening data socket. """ + timeout = 30 def __init__(self, ip, port, cmd_channel): @@ -467,9 +616,9 @@ def __init__(self, ip, port, cmd_channel): self.log_exception = cmd_channel.log_exception self._idler = None if self.timeout: - self._idler = self.ioloop.call_later(self.timeout, - self.handle_timeout, - _errback=self.handle_error) + self._idler = self.ioloop.call_later( + self.timeout, self.handle_timeout, _errback=self.handle_error + ) if ip.count('.') == 3: self._cmd = "PORT" @@ -518,7 +667,8 @@ def handle_timeout(self): msg = "Active data channel timed out." self.cmd_channel.respond("421 " + msg, logfun=logger.info) self.cmd_channel.log_cmd( - self._cmd, self._normalized_addr, 421, msg) + self._cmd, self._normalized_addr, 421, msg + ) self.close() def handle_close(self): @@ -531,7 +681,8 @@ def handle_close(self): msg = "Can't connect to specified address." self.cmd_channel.respond("425 " + msg) self.cmd_channel.log_cmd( - self._cmd, self._normalized_addr, 425, msg) + self._cmd, self._normalized_addr, 425, msg + ) def handle_error(self): """Called to handle any uncaught exceptions.""" @@ -578,9 +729,9 @@ class DTPHandler(AsyncChat): def __init__(self, sock, cmd_channel): """Initialize the command channel. - - (instance) sock: the socket object instance of the newly - established connection. - - (instance) cmd_channel: the command channel class instance. + - (instance) sock: the socket object instance of the newly + established connection. + - (instance) cmd_channel: the command channel class instance. """ self.cmd_channel = cmd_channel self.file_obj = None @@ -607,7 +758,8 @@ def __init__(self, sock, cmd_channel): # instance to set socket attribute before closing, see: # https://github.com/giampaolo/pyftpdlib/issues/188 AsyncChat.__init__( - self, socket.socket(), ioloop=cmd_channel.ioloop) + self, socket.socket(), ioloop=cmd_channel.ioloop + ) # https://github.com/giampaolo/pyftpdlib/issues/143 self.close() if err.errno == errno.EINVAL: @@ -620,13 +772,15 @@ def __init__(self, sock, cmd_channel): self.close() return if self.timeout: - self._idler = self.ioloop.call_every(self.timeout, - self.handle_timeout, - _errback=self.handle_error) + self._idler = self.ioloop.call_every( + self.timeout, self.handle_timeout, _errback=self.handle_error + ) def __repr__(self): - return '<%s(%s)>' % (self.__class__.__name__, - self.cmd_channel.get_repr_info(as_str=True)) + return '<%s(%s)>' % ( + self.__class__.__name__, + self.cmd_channel.get_repr_info(as_str=True), + ) __str__ = __repr__ @@ -682,8 +836,12 @@ def initiate_send(self): def initiate_sendfile(self): """A wrapper around sendfile.""" try: - sent = sendfile(self._fileno, self._filefd, self._offset, - self.ac_out_buffer_size) + sent = sendfile( + self._fileno, + self._filefd, + self._offset, + self.ac_out_buffer_size, + ) except OSError as err: if err.errno in _ERRNOS_RETRY or err.errno == errno.EBUSY: return @@ -692,7 +850,8 @@ def initiate_sendfile(self): else: if self.tot_bytes_sent == 0: logger.warning( - "sendfile() failed; falling back on using plain send") + "sendfile() failed; falling back on using plain send" + ) raise _GiveUpOnSendfile else: raise @@ -882,8 +1041,11 @@ def handle_close(self): self._resp = ("226 Transfer complete.", logger.debug) else: tot_bytes = self.get_transmitted_bytes() - self._resp = ("426 Transfer aborted; %d bytes transmitted." - % tot_bytes, logger.debug) + self._resp = ( + "426 Transfer aborted; %d bytes transmitted." + % tot_bytes, + logger.debug, + ) finally: self.close() @@ -913,7 +1075,8 @@ def close(self): receive=self.receive, completed=self.transfer_finished, elapsed=elapsed_time, - bytes=self.get_transmitted_bytes()) + bytes=self.get_transmitted_bytes(), + ) if self.transfer_finished: if self.receive: self.cmd_channel.on_file_received(filename) @@ -930,9 +1093,12 @@ def close(self): # dirty hack in order to turn AsyncChat into a new style class in # python 2.x so that we can use super() if PY3: + class _AsyncChatNewStyle(AsyncChat): pass + else: + class _AsyncChatNewStyle(object, AsyncChat): # noqa def __init__(self, *args, **kwargs): @@ -956,6 +1122,7 @@ class ThrottledDTPHandler(_AsyncChatNewStyle, DTPHandler): and write limits which results in a less bursty and smoother throughput (default: True). """ + read_limit = 0 write_limit = 0 auto_sized_buffers = True @@ -1019,7 +1186,8 @@ def unsleep(): self.del_channel() self._cancel_throttler() self._throttler = self.ioloop.call_later( - sleepfor, unsleep, _errback=self.handle_error) + sleepfor, unsleep, _errback=self.handle_error + ) self._timenext = now + 1 def close(self): @@ -1038,8 +1206,8 @@ class FileProducer: def __init__(self, file, type): """Initialize the producer with a data_wrapper appropriate to TYPE. - - (file) file: the file[-like] object. - - (str) type: the current TYPE, 'a' (ASCII) or 'i' (binary). + - (file) file: the file[-like] object. + - (str) type: the current TYPE, 'a' (ASCII) or 'i' (binary). """ self.file = file self.type = type @@ -1083,6 +1251,7 @@ def more(self): class BufferedIteratorProducer: """Producer for iterator objects with buffer capabilities.""" + # how many times iterator.next() will be called before # returning some data loops = 20 @@ -1105,6 +1274,7 @@ def more(self): # --- FTP + class FTPHandler(AsyncChat): """Implements the FTP server Protocol Interpreter (see RFC-959), handling commands received from the client on the control channel. @@ -1195,6 +1365,7 @@ class FTPHandler(AsyncChat): - (instance) server: the FTPServer class instance. - (instance) data_channel: the data channel instance (if any). """ + # these are overridable defaults # default classes @@ -1224,9 +1395,9 @@ class FTPHandler(AsyncChat): def __init__(self, conn, server, ioloop=None): """Initialize the command channel. - - (instance) conn: the socket object instance of the newly - established connection. - - (instance) server: the ftp server class instance. + - (instance) conn: the socket object instance of the newly + established connection. + - (instance) server: the ftp server class instance. """ # public session attributes self.server = server @@ -1256,8 +1427,9 @@ def __init__(self, conn, server, ioloop=None): self._current_facts = ['type', 'perm', 'size', 'modify'] self._rnfr = None self._idler = None - self._log_debug = logging.getLogger('pyftpdlib').getEffectiveLevel() \ - <= logging.DEBUG + self._log_debug = ( + logging.getLogger('pyftpdlib').getEffectiveLevel() <= logging.DEBUG + ) if os.name == 'posix': self._current_facts.append('unique') @@ -1287,8 +1459,10 @@ def __init__(self, conn, server, ioloop=None): try: self.remote_ip, self.remote_port = self.socket.getpeername()[:2] except socket.error as err: - debug("call: FTPHandler.__init__, err on getpeername() %r" % err, - self) + debug( + "call: FTPHandler.__init__, err on getpeername() %r" % err, + self, + ) # A race condition may occur if the other end is closing # before we can get the peername, hence ENOTCONN (see issue # #100) while EINVAL can occur on OSX (see issue #143). @@ -1305,8 +1479,9 @@ def __init__(self, conn, server, ioloop=None): try: self.socket.setsockopt(socket.SOL_SOCKET, socket.SO_OOBINLINE, 1) except socket.error as err: - debug("call: FTPHandler.__init__, err on SO_OOBINLINE %r" % err, - self) + debug( + "call: FTPHandler.__init__, err on SO_OOBINLINE %r" % err, self + ) # disable Nagle algorithm for the control socket only, resulting # in significantly better performances @@ -1316,7 +1491,8 @@ def __init__(self, conn, server, ioloop=None): except socket.error as err: debug( "call: FTPHandler.__init__, err on TCP_NODELAY %r" % err, - self) + self, + ) # remove this instance from IOLoop's socket_map if not self.connected: @@ -1325,7 +1501,8 @@ def __init__(self, conn, server, ioloop=None): if self.timeout: self._idler = self.ioloop.call_later( - self.timeout, self.handle_timeout, _errback=self.handle_error) + self.timeout, self.handle_timeout, _errback=self.handle_error + ) def get_repr_info(self, as_str=False, extra_info=None): if extra_info is None: @@ -1447,18 +1624,20 @@ def found_terminator(self): self._in_buffer_len = 0 cmd = line.split(' ')[0].upper() - arg = line[len(cmd) + 1:] + arg = line[len(cmd) + 1 :] try: self.pre_process_command(line, cmd, arg) except UnicodeEncodeError: - self.respond("501 can't decode path (server filesystem encoding " - "is %s)" % sys.getfilesystemencoding()) + self.respond( + "501 can't decode path (server filesystem encoding is %s)" + % sys.getfilesystemencoding() + ) def pre_process_command(self, line, cmd, arg): kwargs = {} if cmd == "SITE" and arg: cmd = "SITE %s" % arg.split(' ')[0].upper() - arg = line[len(cmd) + 1:] + arg = line[len(cmd) + 1 :] if cmd != 'PASS': self.logline("<- %s" % line) @@ -1632,8 +1811,9 @@ def close(self): # actually took place, hence we're not interested in # invoking the callback. if self.remote_ip: - self.ioloop.call_later(0, self.on_disconnect, - _errback=self.handle_error) + self.ioloop.call_later( + 0, self.on_disconnect, _errback=self.handle_error + ) def _shutdown_connecting_dtp(self): """Close any ActiveDTP or PassiveDTP instance waiting to @@ -1752,7 +1932,8 @@ def _on_dtp_close(self): if self._idler is not None and not self._idler.cancelled: self._idler.cancel() self._idler = self.ioloop.call_later( - self.timeout, self.handle_timeout, _errback=self.handle_error) + self.timeout, self.handle_timeout, _errback=self.handle_error + ) # --- utility @@ -1787,7 +1968,8 @@ def push_dtp_data(self, data, isproducer=False, file=None, cmd=None): """ if self.data_channel is not None: self.respond( - "125 Data connection already open. Transfer starting.") + "125 Data connection already open. Transfer starting." + ) if file: self.data_channel.file_obj = file try: @@ -1803,7 +1985,8 @@ def push_dtp_data(self, data, isproducer=False, file=None, cmd=None): self.data_channel.handle_error() else: self.respond( - "150 File status okay. About to open data connection.") + "150 File status okay. About to open data connection." + ) self._out_dtp_queue = (data, isproducer, file, cmd) def flush_account(self): @@ -1871,9 +2054,20 @@ def log_exception(self, instance): # the list of commands which gets logged when logging level # is >= logging.INFO - log_cmds_list = ["DELE", "RNFR", "RNTO", "MKD", "RMD", "CWD", - "XMKD", "XRMD", "XCWD", - "REIN", "SITE CHMOD", "MFMT"] + log_cmds_list = [ + "DELE", + "RNFR", + "RNTO", + "MKD", + "RMD", + "CWD", + "XMKD", + "XRMD", + "XCWD", + "REIN", + "SITE CHMOD", + "MFMT", + ] def log_cmd(self, cmd, arg, respcode, respstr): """Log commands and responses in a standardized format. @@ -1912,27 +2106,32 @@ def log_cmd(self, cmd, arg, respcode, respstr): def log_transfer(self, cmd, filename, receive, completed, elapsed, bytes): """Log all file transfers in a standardized format. - - (str) cmd: - the original command who caused the transfer. + - (str) cmd: + the original command who caused the transfer. - - (str) filename: - the absolutized name of the file on disk. + - (str) filename: + the absolutized name of the file on disk. - - (bool) receive: - True if the transfer was used for client uploading (STOR, - STOU, APPE), False otherwise (RETR). + - (bool) receive: + True if the transfer was used for client uploading (STOR, + STOU, APPE), False otherwise (RETR). - - (bool) completed: - True if the file has been entirely sent, else False. + - (bool) completed: + True if the file has been entirely sent, else False. - - (float) elapsed: - transfer elapsed time in seconds. + - (float) elapsed: + transfer elapsed time in seconds. - - (int) bytes: - number of bytes transmitted. + - (int) bytes: + number of bytes transmitted. """ - line = '%s %s completed=%s bytes=%s seconds=%s' % \ - (cmd, filename, completed and 1 or 0, bytes, elapsed) + line = '%s %s completed=%s bytes=%s seconds=%s' % ( + cmd, + filename, + completed and 1 or 0, + bytes, + elapsed, + ) self.log(line) # --- connection @@ -1955,8 +2154,10 @@ def _make_eport(self, ip, port): # common IPv4 address. remote_ip = remote_ip[7:] if not self.permit_foreign_addresses and ip != remote_ip: - msg = "501 Rejected data connection to foreign address %s:%s." \ - % (ip, port) + msg = "501 Rejected data connection to foreign address %s:%s." % ( + ip, + port, + ) self.respond_w_warning(msg) return @@ -2055,8 +2256,10 @@ def ftp_EPRT(self, line): if af == "1": # test if AF_INET6 and IPV6_V6ONLY - if (self.socket.family == socket.AF_INET6 and not - SUPPORTS_HYBRID_IPV6): + if ( + self.socket.family == socket.AF_INET6 + and not SUPPORTS_HYBRID_IPV6 + ): self.respond('522 Network protocol not supported (use 2).') else: try: @@ -2120,7 +2323,8 @@ def ftp_EPSV(self, line): elif line.lower() == 'all': self._epsvall = True self.respond( - '220 Other commands other than EPSV are now disabled.') + '220 Other commands other than EPSV are now disabled.' + ) else: if self.socket.family == socket.AF_INET: self.respond('501 Unknown network protocol (use 1).') @@ -2243,8 +2447,13 @@ def ftp_MLST(self, path): perms = self.authorizer.get_perms(self.username) try: iterator = self.run_as_current_user( - self.fs.format_mlsx, basedir, [basename], perms, - self._current_facts, ignore_err=False) + self.fs.format_mlsx, + basedir, + [basename], + perms, + self._current_facts, + ignore_err=False, + ) data = b''.join(iterator) except (OSError, FilesystemError) as err: self.respond('550 %s.' % _strerror(err)) @@ -2276,8 +2485,9 @@ def ftp_MLSD(self, path): self.respond('550 %s.' % why) else: perms = self.authorizer.get_perms(self.username) - iterator = self.fs.format_mlsx(path, listing, perms, - self._current_facts) + iterator = self.fs.format_mlsx( + path, listing, perms, self._current_facts + ) producer = BufferedIteratorProducer(iterator) self.push_dtp_data(producer, isproducer=True, cmd="MLSD") return path @@ -2311,7 +2521,9 @@ def ftp_RETR(self, file): ok = 1 except ValueError: why = "REST position (%s) > file size (%s)" % ( - rest_pos, fsize) + rest_pos, + fsize, + ) except (EnvironmentError, FilesystemError) as err: why = _strerror(err) if not ok: @@ -2362,7 +2574,9 @@ def ftp_STOR(self, file, mode='w'): ok = 1 except ValueError: why = "REST position (%s) > file size (%s)" % ( - rest_pos, fsize) + rest_pos, + fsize, + ) except (EnvironmentError, FilesystemError) as err: why = _strerror(err) if not ok: @@ -2410,8 +2624,9 @@ def ftp_STOU(self, line): basedir = self.fs.ftp2fs(self.fs.cwd) prefix = 'ftpd.' try: - fd = self.run_as_current_user(self.fs.mkstemp, prefix=prefix, - dir=basedir) + fd = self.run_as_current_user( + self.fs.mkstemp, prefix=prefix, dir=basedir + ) except (EnvironmentError, FilesystemError) as err: # likely, we hit the max number of retries to find out a # file with a unique name @@ -2475,15 +2690,19 @@ def ftp_REST(self, line): def ftp_ABOR(self, line): """Abort the current data transfer.""" # ABOR received while no data channel exists - if self._dtp_acceptor is None and \ - self._dtp_connector is None and \ - self.data_channel is None: + if ( + self._dtp_acceptor is None + and self._dtp_connector is None + and self.data_channel is None + ): self.respond("225 No transfer to abort.") return else: # a PASV or PORT was received but connection wasn't made yet - if self._dtp_acceptor is not None or \ - self._dtp_connector is not None: + if ( + self._dtp_acceptor is not None + or self._dtp_connector is not None + ): self._shutdown_connecting_dtp() resp = "225 ABOR command successful; data channel closed." @@ -2498,8 +2717,9 @@ def ftp_ABOR(self, line): if self.data_channel.transfer_in_progress(): self.data_channel.close() self.data_channel = None - self.respond("426 Transfer aborted via ABOR.", - logfun=logger.info) + self.respond( + "426 Transfer aborted via ABOR.", logfun=logger.info + ) resp = "226 ABOR command successful." else: self.data_channel.close() @@ -2508,6 +2728,7 @@ def ftp_ABOR(self, line): self.respond(resp) # --- authentication + def ftp_USER(self, line): """Set the username for the current session.""" # RFC-959 specifies a 530 response to the USER command if the @@ -2552,9 +2773,14 @@ def callback(username, password, msg): else: # response string should be capitalized as per RFC-959 msg = msg.capitalize() - self.ioloop.call_later(self.auth_failed_timeout, callback, - self.username, password, msg, - _errback=self.handle_error) + self.ioloop.call_later( + self.auth_failed_timeout, + callback, + self.username, + password, + msg, + _errback=self.handle_error, + ) self.username = "" def handle_auth_success(self, home, password, msg_login): @@ -2564,9 +2790,11 @@ def handle_auth_success(self, home, password, msg_login): else: warnings.warn( '%s.get_home_dir returned a non-unicode string; now ' - 'casting to unicode' % ( - self.authorizer.__class__.__name__), - RuntimeWarning, stacklevel=2) + 'casting to unicode' + % (self.authorizer.__class__.__name__), + RuntimeWarning, + stacklevel=2, + ) home = home.decode('utf8') if len(msg_login) <= 75: @@ -2615,6 +2843,7 @@ def ftp_REIN(self, line): self.respond("230 Ready for new user.") # --- filesystem operations + def ftp_PWD(self, line): """Return the name of the current working directory to the client.""" # The 257 response is supposed to include the directory @@ -2622,8 +2851,9 @@ def ftp_PWD(self, line): # they must be doubled (see RFC-959, chapter 7, appendix 2). cwd = self.fs.cwd assert isinstance(cwd, unicode), cwd - self.respond('257 "%s" is the current directory.' - % cwd.replace('"', '""')) + self.respond( + '257 "%s" is the current directory.' % cwd.replace('"', '""') + ) def ftp_CWD(self, path): """Change the current working directory. @@ -2717,7 +2947,7 @@ def ftp_MDTM(self, path): return path def ftp_MFMT(self, path, timeval): - """ Sets the last modification time of file to timeval + """Sets the last modification time of file to timeval 3307 style timestamp (YYYYMMDDHHMMSS) as defined in RFC-3659. On success return the modified time and file path, else None. """ @@ -2778,7 +3008,8 @@ def ftp_MKD(self, path): # name and in case it contains embedded double-quotes # they must be doubled (see RFC-959, chapter 7, appendix 2). self.respond( - '257 "%s" directory created.' % line.replace('"', '""')) + '257 "%s" directory created.' % line.replace('"', '""') + ) return path def ftp_RMD(self, path): @@ -2841,6 +3072,7 @@ def ftp_RNTO(self, path): return (src, path) # --- others + def ftp_TYPE(self, line): """Set current type data type to binary/ascii.""" type = line.upper().replace(' ', '') @@ -2963,9 +3195,11 @@ def ftp_STAT(self, path): def ftp_FEAT(self, line): """List all new features supported as defined in RFC-2398.""" features = set(['UTF8', 'TVFS']) - features.update([feat for feat in - ('EPRT', 'EPSV', 'MDTM', 'MFMT', 'SIZE') - if feat in self.proto_cmds]) + features.update([ + feat + for feat in ('EPRT', 'EPSV', 'MDTM', 'MFMT', 'SIZE') + if feat in self.proto_cmds + ]) features.update(self._extra_feats) if 'MLST' in self.proto_cmds or 'MLSD' in self.proto_cmds: facts = '' @@ -3000,8 +3234,9 @@ def ftp_OPTS(self, line): self.respond('501 %s.' % err) else: facts = [x.lower() for x in arg.split(';')] - self._current_facts = \ - [x for x in facts if x in self._available_facts] + self._current_facts = [ + x for x in facts if x in self._available_facts + ] f = ''.join([x + ';' for x in self._current_facts]) self.respond('200 MLST OPTS ' + f) @@ -3035,8 +3270,9 @@ def ftp_HELP(self, line): # provide a compact list of recognized commands def formatted_help(): cmds = [] - keys = sorted([x for x in self.proto_cmds - if not x.startswith('SITE ')]) + keys = sorted( + [x for x in self.proto_cmds if not x.startswith('SITE ')] + ) while keys: elems = tuple(keys[0:8]) cmds.append(' %-6s' * len(elems) % elems + '\r\n') @@ -3145,9 +3381,11 @@ def __init__(self, *args, **kwargs): self._ssl_want_write = False def readable(self): - return self._ssl_accepting or \ - self._ssl_want_read or \ - super().readable() + return ( + self._ssl_accepting + or self._ssl_want_read + or super().readable() + ) def writable(self): return self._ssl_want_write or super().writable() @@ -3165,14 +3403,17 @@ def secure_connection(self, ssl_context): # very quickly debug( "call: secure_connection(); can't secure SSL connection " - "%r; closing" % err, self) + "%r; closing" % err, + self, + ) self.close() except ValueError: # may happen in case the client connects/disconnects # very quickly if self.socket.fileno() == -1: debug( - "ValueError and fd == -1 on secure_connection()", self) + "ValueError and fd == -1 on secure_connection()", self + ) return raise else: @@ -3193,10 +3434,12 @@ def _handle_ssl_want_rw(self): if self._ssl_want_read: self.modify_ioloop_events( - self._wanted_io_events | self.ioloop.READ, logdebug=True) + self._wanted_io_events | self.ioloop.READ, logdebug=True + ) elif self._ssl_want_write: self.modify_ioloop_events( - self._wanted_io_events | self.ioloop.WRITE, logdebug=True) + self._wanted_io_events | self.ioloop.WRITE, logdebug=True + ) else: if prev_row_pending: self.modify_ioloop_events(self._wanted_io_events) @@ -3212,8 +3455,9 @@ def _do_ssl_handshake(self): debug("call: _do_ssl_handshake, err: ssl-want-read", inst=self) except SSL.WantWriteError: self._ssl_want_write = True - debug("call: _do_ssl_handshake, err: ssl-want-write", - inst=self) + debug( + "call: _do_ssl_handshake, err: ssl-want-write", inst=self + ) except SSL.SysCallError as err: debug("call: _do_ssl_handshake, err: %r" % err, inst=self) retval, desc = err.args @@ -3302,7 +3546,8 @@ def send(self, data): return 0 except SSL.ZeroReturnError: debug( - "call: send() -> shutdown(), err: zero-return", inst=self) + "call: send() -> shutdown(), err: zero-return", inst=self + ) super().handle_close() return 0 except SSL.SysCallError as err: @@ -3310,8 +3555,10 @@ def send(self, data): errnum, errstr = err.args if errnum == errno.EWOULDBLOCK: return 0 - elif errnum in _ERRNOS_DISCONNECTED or \ - errstr == 'Unexpected EOF': + elif ( + errnum in _ERRNOS_DISCONNECTED + or errstr == 'Unexpected EOF' + ): super().handle_close() return 0 else: @@ -3329,15 +3576,18 @@ def recv(self, buffer_size): self._ssl_want_write = True raise RetryError except SSL.ZeroReturnError: - debug("call: recv() -> shutdown(), err: zero-return", - inst=self) + debug( + "call: recv() -> shutdown(), err: zero-return", inst=self + ) super().handle_close() return b'' except SSL.SysCallError as err: debug("call: recv(), err: %r" % err, inst=self) errnum, errstr = err.args - if errnum in _ERRNOS_DISCONNECTED or \ - errstr == 'Unexpected EOF': + if ( + errnum in _ERRNOS_DISCONNECTED + or errstr == 'Unexpected EOF' + ): super().handle_close() return b'' else: @@ -3358,9 +3608,13 @@ def _do_ssl_shutdown(self): except (OSError, socket.error) as err: debug( "call: _do_ssl_shutdown() -> os.write, err: %r" % err, - inst=self) - if err.errno in (errno.EINTR, errno.EWOULDBLOCK, - errno.ENOBUFS): + inst=self, + ) + if err.errno in { + errno.EINTR, + errno.EWOULDBLOCK, + errno.ENOBUFS, + }: return elif err.errno in _ERRNOS_DISCONNECTED: return super().close() @@ -3398,20 +3652,27 @@ def _do_ssl_shutdown(self): except SSL.ZeroReturnError: debug( "call: _do_ssl_shutdown() -> shutdown(), err: zero-return", - inst=self) + inst=self, + ) super().close() except SSL.SysCallError as err: - debug("call: _do_ssl_shutdown() -> shutdown(), err: %r" % err, - inst=self) + debug( + "call: _do_ssl_shutdown() -> shutdown(), err: %r" % err, + inst=self, + ) errnum, errstr = err.args - if errnum in _ERRNOS_DISCONNECTED or \ - errstr == 'Unexpected EOF': + if ( + errnum in _ERRNOS_DISCONNECTED + or errstr == 'Unexpected EOF' + ): super().close() else: raise except SSL.Error as err: - debug("call: _do_ssl_shutdown() -> shutdown(), err: %r" % err, - inst=self) + debug( + "call: _do_ssl_shutdown() -> shutdown(), err: %r" % err, + inst=self, + ) # see: # https://github.com/giampaolo/pyftpdlib/issues/171 # https://bugs.launchpad.net/pyopenssl/+bug/785985 @@ -3420,23 +3681,28 @@ def _do_ssl_shutdown(self): else: raise except socket.error as err: - debug("call: _do_ssl_shutdown() -> shutdown(), err: %r" % err, - inst=self) + debug( + "call: _do_ssl_shutdown() -> shutdown(), err: %r" % err, + inst=self, + ) if err.errno in _ERRNOS_DISCONNECTED: super().close() else: raise else: if done: - debug("call: _do_ssl_shutdown(), shutdown completed", - inst=self) + debug( + "call: _do_ssl_shutdown(), shutdown completed", + inst=self, + ) self._ssl_established = False self._ssl_closing = False self.handle_ssl_shutdown() else: debug( "call: _do_ssl_shutdown(), shutdown not completed yet", - inst=self) + inst=self, + ) def close(self): if self._ssl_established and not self._error: @@ -3541,16 +3807,28 @@ class TLS_FTPHandler(SSLConnection, FTPHandler): proto_cmds = FTPHandler.proto_cmds.copy() proto_cmds.update({ 'AUTH': dict( - perm=None, auth=False, arg=True, - help='Syntax: AUTH TLS|SSL (set up secure control ' - 'channel).'), + perm=None, + auth=False, + arg=True, + help=( + 'Syntax: AUTH TLS|SSL (set up secure control ' + 'channel).' + ), + ), 'PBSZ': dict( - perm=None, auth=False, arg=True, - help='Syntax: PBSZ 0 (negotiate TLS buffer).'), + perm=None, + auth=False, + arg=True, + help='Syntax: PBSZ 0 (negotiate TLS buffer).', + ), 'PROT': dict( - perm=None, auth=False, arg=True, - help='Syntax: PROT [C|P] (set up un/secure data ' - 'channel).'), + perm=None, + auth=False, + arg=True, + help=( + 'Syntax: PROT [C|P] (set up un/secure data channel).' + ), + ), }) def __init__(self, conn, server, ioloop=None): @@ -3628,7 +3906,8 @@ def ftp_AUTH(self, line): self.secure_connection(self.ssl_context) else: self.respond( - "502 Unrecognized encryption type (use TLS or SSL).") + "502 Unrecognized encryption type (use TLS or SSL)." + ) def ftp_PBSZ(self, line): """Negotiate size of buffer for secure data transfer. @@ -3637,7 +3916,8 @@ def ftp_PBSZ(self, line): """ if not isinstance(self.socket, SSL.Connection): self.respond( - "503 PBSZ not allowed on insecure control connection.") + "503 PBSZ not allowed on insecure control connection." + ) else: self.respond('200 PBSZ=0 successful.') self._pbsz = True @@ -3647,10 +3927,12 @@ def ftp_PROT(self, line): arg = line.upper() if not isinstance(self.socket, SSL.Connection): self.respond( - "503 PROT not allowed on insecure control connection.") + "503 PROT not allowed on insecure control connection." + ) elif not self._pbsz: self.respond( - "503 You must issue the PBSZ command prior to PROT.") + "503 You must issue the PBSZ command prior to PROT." + ) elif arg == 'C': self.respond('200 Protection set to Clear') self._prot = False diff --git a/pyftpdlib/ioloop.py b/pyftpdlib/ioloop.py index b7a1b25d..43d1651a 100644 --- a/pyftpdlib/ioloop.py +++ b/pyftpdlib/ioloop.py @@ -92,8 +92,14 @@ def handle_accepted(self, sock, addr): # These errnos indicate that a connection has been abruptly terminated. _ERRNOS_DISCONNECTED = set(( - errno.ECONNRESET, errno.ENOTCONN, errno.ESHUTDOWN, errno.ECONNABORTED, - errno.EPIPE, errno.EBADF, errno.ETIMEDOUT)) + errno.ECONNRESET, + errno.ENOTCONN, + errno.ESHUTDOWN, + errno.ECONNABORTED, + errno.EPIPE, + errno.EBADF, + errno.ETIMEDOUT, +)) if hasattr(errno, "WSAECONNRESET"): _ERRNOS_DISCONNECTED.add(errno.WSAECONNRESET) if hasattr(errno, "WSAECONNABORTED"): @@ -114,6 +120,7 @@ class RetryError(Exception): # --- scheduler # =================================================================== + class _Scheduler: """Run the scheduled functions due to expire soonest (if any).""" @@ -150,8 +157,9 @@ def poll(self): # remove cancelled tasks and re-heapify the queue if the # number of cancelled tasks is more than the half of the # entire queue - if self._cancellations > 512 and \ - self._cancellations > (len(self._tasks) >> 1): + if self._cancellations > 512 and self._cancellations > ( + len(self._tasks) >> 1 + ): debug("re-heapifying %s cancelled tasks" % self._cancellations) self.reheapify() @@ -180,13 +188,23 @@ def reheapify(self): class _CallLater: """Container object which instance is returned by ioloop.call_later().""" - __slots__ = ('_delay', '_target', '_args', '_kwargs', '_errback', '_sched', - '_repush', 'timeout', 'cancelled') + __slots__ = ( + '_delay', + '_target', + '_args', + '_kwargs', + '_errback', + '_sched', + '_repush', + 'timeout', + 'cancelled', + ) def __init__(self, seconds, target, *args, **kwargs): assert callable(target), "%s is not callable" % target - assert sys.maxsize >= seconds >= 0, \ + assert sys.maxsize >= seconds >= 0, ( "%s is not greater than or equal to 0 seconds" % seconds + ) self._delay = seconds self._target = target self._args = args @@ -214,8 +232,11 @@ def __repr__(self): else: sig = repr(self._target) sig += ' args=%s, kwargs=%s, cancelled=%s, secs=%s' % ( - self._args or '[]', self._kwargs or '{}', self.cancelled, - self._delay) + self._args or '[]', + self._kwargs or '{}', + self.cancelled, + self._delay, + ) return '<%s>' % sig __str__ = __repr__ @@ -286,8 +307,10 @@ def __exit__(self, *args): def __repr__(self): status = [self.__class__.__module__ + "." + self.__class__.__name__] - status.append("(fds=%s, tasks=%s)" % ( - len(self.socket_map), len(self.sched._tasks))) + status.append( + "(fds=%s, tasks=%s)" + % (len(self.socket_map), len(self.sched._tasks)) + ) return '<%s at %#x>' % (' '.join(status), id(self)) __str__ = __repr__ @@ -328,13 +351,13 @@ def poll(self, timeout): def loop(self, timeout=None, blocking=True): """Start the asynchronous IO loop. - - (float) timeout: the timeout passed to the underlying - multiplex syscall (select(), epoll() etc.). + - (float) timeout: the timeout passed to the underlying + multiplex syscall (select(), epoll() etc.). - - (bool) blocking: if True poll repeatedly, as long as there - are registered handlers and/or scheduled functions. - If False poll only once and return the timeout of the next - scheduled call (if any, else None). + - (bool) blocking: if True poll repeatedly, as long as there + are registered handlers and/or scheduled functions. + If False poll only once and return the timeout of the next + scheduled call (if any, else None). """ if not _IOLoop._started_once: _IOLoop._started_once = True @@ -379,7 +402,7 @@ def call_later(self, seconds, target, *args, **kwargs): - kwargs: the keyword arguments to call it with; a special '_errback' parameter can be passed: it is a callable called in case target function raises an exception. - """ + """ kwargs['_scheduler'] = self.sched return _CallLater(seconds, target, *args, **kwargs) @@ -419,6 +442,7 @@ def close(self): # --- select() - POSIX / Windows # =================================================================== + class Select(_IOLoop): """select()-based poller.""" @@ -479,6 +503,7 @@ def poll(self, timeout): # --- poll() / epoll() # =================================================================== + class _BasePollEpoll(_IOLoop): """This is common to both poll() (UNIX), epoll() (Linux) and /dev/poll (Solaris) implementations which share almost the same @@ -510,8 +535,11 @@ def unregister(self, fd): self._poller.unregister(fd) except EnvironmentError as err: if err.errno in (errno.ENOENT, errno.EBADF): - debug("call: unregister(); poller returned %r; " - "ignoring it" % err, self) + debug( + "call: unregister(); poller returned %r; ignoring it" + % err, + self, + ) else: raise @@ -594,6 +622,7 @@ class DevPoll(_BasePollEpoll): # introduced in python 3.4 if hasattr(select.devpoll, 'fileno'): + def fileno(self): """Return devpoll() fd.""" return self._poller.fileno() @@ -611,6 +640,7 @@ def poll(self, timeout): # introduced in python 3.4 if hasattr(select.devpoll, 'close'): + def close(self): _IOLoop.close(self) self._poller.close() @@ -667,8 +697,9 @@ def register(self, fd, instance, events): self._control(fd, events, select.KQ_EV_ADD) except EnvironmentError as err: if err.errno == errno.EEXIST: - debug("call: register(); poller raised EEXIST; ignored", - self) + debug( + "call: register(); poller raised EEXIST; ignored", self + ) else: raise self._active[fd] = events @@ -684,8 +715,11 @@ def unregister(self, fd): self._control(fd, events, select.KQ_EV_DELETE) except EnvironmentError as err: if err.errno in (errno.ENOENT, errno.EBADF): - debug("call: unregister(); poller returned %r; " - "ignoring it" % err, self) + debug( + "call: unregister(); poller returned %r; " + "ignoring it" % err, + self, + ) else: raise @@ -697,12 +731,18 @@ def modify(self, fd, events): def _control(self, fd, events, flags): kevents = [] if events & self.WRITE: - kevents.append(select.kevent( - fd, filter=select.KQ_FILTER_WRITE, flags=flags)) + kevents.append( + select.kevent( + fd, filter=select.KQ_FILTER_WRITE, flags=flags + ) + ) if events & self.READ or not kevents: # always read when there is not a write - kevents.append(select.kevent( - fd, filter=select.KQ_FILTER_READ, flags=flags)) + kevents.append( + select.kevent( + fd, filter=select.KQ_FILTER_READ, flags=flags + ) + ) # even though control() takes a list, it seems to return # EINVAL on Mac OS X (10.6) when there is more than one # event in the list @@ -710,16 +750,19 @@ def _control(self, fd, events, flags): self._kqueue.control([kevent], 0) # localize variable access to minimize overhead - def poll(self, - timeout, - _len=len, - _READ=select.KQ_FILTER_READ, - _WRITE=select.KQ_FILTER_WRITE, - _EOF=select.KQ_EV_EOF, - _ERROR=select.KQ_EV_ERROR): + def poll( + self, + timeout, + _len=len, + _READ=select.KQ_FILTER_READ, + _WRITE=select.KQ_FILTER_WRITE, + _EOF=select.KQ_EV_EOF, + _ERROR=select.KQ_EV_ERROR, + ): try: - kevents = self._kqueue.control(None, _len(self.socket_map), - timeout) + kevents = self._kqueue.control( + None, _len(self.socket_map), timeout + ) except OSError as err: if err.errno == errno.EINTR: return @@ -751,15 +794,15 @@ def poll(self, # --- choose the better poller for this platform # =================================================================== -if hasattr(select, 'epoll'): # epoll() - Linux +if hasattr(select, 'epoll'): # epoll() - Linux IOLoop = Epoll -elif hasattr(select, 'kqueue'): # kqueue() - BSD / OSX +elif hasattr(select, 'kqueue'): # kqueue() - BSD / OSX IOLoop = Kqueue elif hasattr(select, 'devpoll'): # /dev/poll - Solaris IOLoop = DevPoll -elif hasattr(select, 'poll'): # poll() - POSIX +elif hasattr(select, 'poll'): # poll() - POSIX IOLoop = Poll -else: # select() - POSIX and Windows +else: # select() - POSIX and Windows IOLoop = Select @@ -806,7 +849,9 @@ def modify_ioloop_events(self, events, logdebug=False): if self._fileno not in self.ioloop.socket_map: debug( "call: modify_ioloop_events(), fd was no longer in " - "socket_map, had to register() it again", inst=self) + "socket_map, had to register() it again", + inst=self, + ) self.add_channel(events=events) else: if events != self._current_io_events: @@ -819,14 +864,19 @@ def modify_ioloop_events(self, events, logdebug=False): ev = "RW" else: ev = events - debug("call: IOLoop.modify(); setting %r IO events" % ( - ev), self) + debug( + "call: IOLoop.modify(); setting %r IO events" + % (ev), + self, + ) self.ioloop.modify(self._fileno, events) self._current_io_events = events else: debug( "call: modify_ioloop_events(), handler had already been " - "close()d, skipping modify()", inst=self) + "close()d, skipping modify()", + inst=self, + ) # --- utils @@ -853,8 +903,14 @@ def connect_af_unspecified(self, addr, source_address=None): assert self.socket is None host, port = addr err = "getaddrinfo() returned an empty list" - info = socket.getaddrinfo(host, port, socket.AF_UNSPEC, - socket.SOCK_STREAM, 0, socket.AI_PASSIVE) + info = socket.getaddrinfo( + host, + port, + socket.AF_UNSPEC, + socket.SOCK_STREAM, + 0, + socket.AI_PASSIVE, + ) for res in info: self.socket = None af, socktype, proto, canonname, sa = res @@ -871,8 +927,10 @@ def connect_af_unspecified(self, addr, source_address=None): # http://tools.ietf.org/html/rfc3493.html#section-3.7 # We truncate the first bytes to make it look like a # common IPv4 address. - source_address = (source_address[0][7:], - source_address[1]) + source_address = ( + source_address[0][7:], + source_address[1], + ) self.bind(source_address) self.connect((host, port)) except socket.error as _: @@ -950,8 +1008,9 @@ def initiate_send(self): self.ioloop.modify(self._fileno, wanted) self._wanted_io_events = wanted else: - debug("call: initiate_send(); called with no connection", - inst=self) + debug( + "call: initiate_send(); called with no connection", inst=self + ) def close_when_done(self): if len(self.producer_fifo) == 0: @@ -1007,8 +1066,14 @@ def bind_af_unspecified(self, addr): # instead of "", so we'll make the conversion for them. host = None err = "getaddrinfo() returned an empty list" - info = socket.getaddrinfo(host, port, socket.AF_UNSPEC, - socket.SOCK_STREAM, 0, socket.AI_PASSIVE) + info = socket.getaddrinfo( + host, + port, + socket.AF_UNSPEC, + socket.SOCK_STREAM, + 0, + socket.AI_PASSIVE, + ) for res in info: self.socket = None self.del_channel() @@ -1054,8 +1119,10 @@ def handle_accept(self): if err.errno != errno.ECONNABORTED: raise else: - debug("call: handle_accept(); accept() returned ECONNABORTED", - self) + debug( + "call: handle_accept(); accept() returned ECONNABORTED", + self, + ) else: # sometimes addr == None instead of (ip, port) (see issue 104) if addr is not None: @@ -1067,5 +1134,6 @@ def handle_accepted(self, sock, addr): # overridden for convenience; avoid to reuse address on Windows if (os.name in ('nt', 'ce')) or (sys.platform == 'cygwin'): + def set_reuse_addr(self): pass diff --git a/pyftpdlib/log.py b/pyftpdlib/log.py index a7bfb46a..c972dbbb 100644 --- a/pyftpdlib/log.py +++ b/pyftpdlib/log.py @@ -58,6 +58,7 @@ class LogFormatter(logging.Formatter): * Timestamps on every log line. * Robust against str/bytes encoding problems. """ + PREFIX = PREFIX def __init__(self, *args, **kwargs): @@ -72,8 +73,9 @@ def __init__(self, *args, **kwargs): # works with unicode strings. The explicit calls to # unicode() below are harmless in python2 but will do the # right conversion in python 3. - fg_color = \ + fg_color = ( curses.tigetstr("setaf") or curses.tigetstr("setf") or "" + ) if not PY3: fg_color = unicode(fg_color, "ascii") self._colors = { @@ -84,7 +86,7 @@ def __init__(self, *args, **kwargs): # yellow logging.WARNING: unicode(curses.tparm(fg_color, 3), "ascii"), # red - logging.ERROR: unicode(curses.tparm(fg_color, 1), "ascii") + logging.ERROR: unicode(curses.tparm(fg_color, 1), "ascii"), } self._normal = unicode(curses.tigetstr("sgr0"), "ascii") @@ -94,12 +96,16 @@ def format(self, record): except Exception as err: record.message = "Bad message (%r): %r" % (err, record.__dict__) - record.asctime = time.strftime(TIME_FORMAT, - self.converter(record.created)) + record.asctime = time.strftime( + TIME_FORMAT, self.converter(record.created) + ) prefix = self.PREFIX % record.__dict__ if self._coloured: - prefix = self._colors.get(record.levelno, self._normal) + \ - prefix + self._normal + prefix = ( + self._colors.get(record.levelno, self._normal) + + prefix + + self._normal + ) # Encoding notes: The logging module prefers to work with character # strings, but only enforces that log messages are instances of @@ -147,24 +153,30 @@ def is_logging_configured(): # TODO: write tests + def config_logging(level=LEVEL, prefix=PREFIX, other_loggers=None): # Speedup logging by preventing certain internal log record info to # be unnecessarily fetched. This results in about 28% speedup. See: # * https://docs.python.org/3/howto/logging.html#optimization # * https://docs.python.org/3/library/logging.html#logrecord-attributes # * https://stackoverflow.com/a/38924153/376587 - key_names = set(re.findall( - r'(?, dispatching the requests to a (typically FTPHandler class). @@ -155,24 +156,29 @@ def get_fqname(obj): config_logging(prefix=PREFIX_MPROC if prefork else PREFIX) if self.handler.passive_ports: - pasv_ports = "%s->%s" % (self.handler.passive_ports[0], - self.handler.passive_ports[-1]) + pasv_ports = "%s->%s" % ( + self.handler.passive_ports[0], + self.handler.passive_ports[-1], + ) else: pasv_ports = None model = 'prefork + ' if prefork else '' - if 'ThreadedFTPServer' in __all__ and \ - issubclass(self.__class__, ThreadedFTPServer): + if 'ThreadedFTPServer' in __all__ and issubclass( + self.__class__, ThreadedFTPServer + ): model += 'multi-thread' - elif 'MultiprocessFTPServer' in __all__ and \ - issubclass(self.__class__, MultiprocessFTPServer): + elif 'MultiprocessFTPServer' in __all__ and issubclass( + self.__class__, MultiprocessFTPServer + ): model += 'multi-process' elif issubclass(self.__class__, FTPServer): model += 'async' else: model += 'unknown (custom class)' logger.info("concurrency model: " + model) - logger.info("masquerade (NAT) address: %s", - self.handler.masquerade_address) + logger.info( + "masquerade (NAT) address: %s", self.handler.masquerade_address + ) logger.info("passive ports: %s", pasv_ports) logger.debug("poller: %r", get_fqname(self.ioloop)) logger.debug("authorizer: %r", get_fqname(self.handler.authorizer)) @@ -180,8 +186,9 @@ def get_fqname(obj): logger.debug("use sendfile(2): %s", self.handler.use_sendfile) logger.debug("handler: %r", get_fqname(self.handler)) logger.debug("max connections: %s", self.max_cons or "unlimited") - logger.debug("max connections per ip: %s", - self.max_cons_per_ip or "unlimited") + logger.debug( + "max connections per ip: %s", self.max_cons_per_ip or "unlimited" + ) logger.debug("timeout: %s", self.handler.timeout or "unlimited") logger.debug("banner: %r", self.handler.banner) logger.debug("max login attempts: %r", self.handler.max_login_attempts) @@ -190,34 +197,35 @@ def get_fqname(obj): if getattr(self.handler, 'keyfile', None): logger.debug("SSL keyfile: %r", self.handler.keyfile) - def serve_forever(self, timeout=None, blocking=True, handle_exit=True, - worker_processes=1): + def serve_forever( + self, timeout=None, blocking=True, handle_exit=True, worker_processes=1 + ): """Start serving. - - (float) timeout: the timeout passed to the underlying IO - loop expressed in seconds. - - - (bool) blocking: if False loop once and then return the - timeout of the next scheduled call next to expire soonest - (if any). - - - (bool) handle_exit: when True catches KeyboardInterrupt and - SystemExit exceptions (generally caused by SIGTERM / SIGINT - signals) and gracefully exits after cleaning up resources. - Also, logs server start and stop. - - - (int) worker_processes: pre-fork a certain number of child - processes before starting. - Each child process will keep using a 1-thread, async - concurrency model, handling multiple concurrent connections. - If the number is None or <= 0 the number of usable cores - available on this machine is detected and used. - It is a good idea to use this option in case the app risks - blocking for too long on a single function call (e.g. - hard-disk is slow, long DB query on auth etc.). - By splitting the work load over multiple processes the delay - introduced by a blocking function call is amortized and divided - by the number of worker processes. + - (float) timeout: the timeout passed to the underlying IO + loop expressed in seconds. + + - (bool) blocking: if False loop once and then return the + timeout of the next scheduled call next to expire soonest + (if any). + + - (bool) handle_exit: when True catches KeyboardInterrupt and + SystemExit exceptions (generally caused by SIGTERM / SIGINT + signals) and gracefully exits after cleaning up resources. + Also, logs server start and stop. + + - (int) worker_processes: pre-fork a certain number of child + processes before starting. + Each child process will keep using a 1-thread, async + concurrency model, handling multiple concurrent connections. + If the number is None or <= 0 the number of usable cores + available on this machine is detected and used. + It is a good idea to use this option in case the app risks + blocking for too long on a single function call (e.g. + hard-disk is slow, long DB query on auth etc.). + By splitting the work load over multiple processes the delay + introduced by a blocking function call is amortized and divided + by the number of worker processes. """ log = handle_exit and blocking @@ -225,7 +233,8 @@ def serve_forever(self, timeout=None, blocking=True, handle_exit=True, if worker_processes != 1 and os.name == 'posix': if not blocking: raise ValueError( - "'worker_processes' and 'blocking' are mutually exclusive") + "'worker_processes' and 'blocking' are mutually exclusive" + ) if log: self._log_start(prefork=True) fork_processes(worker_processes) @@ -235,8 +244,10 @@ def serve_forever(self, timeout=None, blocking=True, handle_exit=True, # proto = "FTP+SSL" if hasattr(self.handler, 'ssl_protocol') else "FTP" - logger.info(">>> starting %s server on %s:%s, pid=%i <<<" - % (proto, self.address[0], self.address[1], os.getpid())) + logger.info( + ">>> starting %s server on %s:%s, pid=%i <<<" + % (proto, self.address[0], self.address[1], os.getpid()) + ) # if handle_exit: @@ -248,7 +259,10 @@ def serve_forever(self, timeout=None, blocking=True, handle_exit=True, if log: logger.info( ">>> shutting down FTP server, %s socket(s), pid=%i " - "<<<", self._map_len(), os.getpid()) + "<<<", + self._map_len(), + os.getpid(), + ) self.close_all() else: self.ioloop.loop(timeout, blocking) @@ -321,6 +335,7 @@ def close_all(self): # --- extra implementations # =================================================================== + class _SpawnerBase(FTPServer): """Base class shared by multiple threads/process dispatcher. Not supposed to be used. @@ -335,13 +350,15 @@ class _SpawnerBase(FTPServer): _exit = None def __init__(self, address_or_socket, handler, ioloop=None, backlog=100): - FTPServer.__init__(self, address_or_socket, handler, - ioloop=ioloop, backlog=backlog) + FTPServer.__init__( + self, address_or_socket, handler, ioloop=ioloop, backlog=backlog + ) self._active_tasks = [] self._active_tasks_idler = self.ioloop.call_every( self.refresh_interval, self._refresh_tasks, - _errback=self.handle_error) + _errback=self.handle_error, + ) def _start_task(self, *args, **kwargs): raise NotImplementedError('must be implemented in subclass') @@ -360,8 +377,10 @@ def _refresh_tasks(self): This gets called every X secs. """ if self._active_tasks: - logger.debug("refreshing tasks (%s join() potentials)" % - len(self._active_tasks)) + logger.debug( + "refreshing tasks (%s join() potentials)" + % len(self._active_tasks) + ) with self._lock: new = [] for t in self._active_tasks: @@ -382,8 +401,9 @@ def _loop(self, handler): if err.errno == errno.EBADF: # we might get here in case the other end quickly # disconnected (see test_quick_connect()) - debug("call: %s._loop(); add_channel() returned EBADF", - self) + debug( + "call: %s._loop(); add_channel() returned EBADF", self + ) return else: raise @@ -394,8 +414,9 @@ def _loop(self, handler): poll_timeout = getattr(self, 'poll_timeout', None) soonest_timeout = poll_timeout - while (ioloop.socket_map or ioloop.sched._tasks) and \ - not self._exit.is_set(): + while ( + ioloop.socket_map or ioloop.sched._tasks + ) and not self._exit.is_set(): try: if ioloop.socket_map: poll(timeout=soonest_timeout) @@ -430,8 +451,10 @@ def _loop(self, handler): select.select([fd], [], [], 0) except select.error: try: - logger.info("discarding broken socket %r", - ioloop.socket_map[fd]) + logger.info( + "discarding broken socket %r", + ioloop.socket_map[fd], + ) del ioloop.socket_map[fd] except KeyError: # dict changed during iteration @@ -440,8 +463,10 @@ def _loop(self, handler): raise else: if poll_timeout: - if soonest_timeout is None or \ - soonest_timeout > poll_timeout: + if ( + soonest_timeout is None + or soonest_timeout > poll_timeout + ): soonest_timeout = poll_timeout def handle_accepted(self, sock, addr): @@ -451,8 +476,9 @@ def handle_accepted(self, sock, addr): # main thread to accept connections self.ioloop.unregister(handler._fileno) - t = self._start_task(target=self._loop, args=(handler, ), - name='ftpd') + t = self._start_task( + target=self._loop, args=(handler,), name='ftpd' + ) t.name = repr(addr) t.start() @@ -481,7 +507,8 @@ def serve_forever(self, timeout=1.0, blocking=True, handle_exit=True): if log: logger.info( ">>> shutting down FTP server (%s active workers) <<<", - self._map_len()) + self._map_len(), + ) self.close_all() else: self.ioloop.loop(timeout, blocking) @@ -505,8 +532,9 @@ def _join_task(self, t): logger.debug("join()ing task %r" % t) t.join(self.join_timeout) if t.is_alive(): - logger.warning("task %r remained alive after %r secs", t, - self.join_timeout) + logger.warning( + "task %r remained alive after %r secs", t, self.join_timeout + ) def close_all(self): self._active_tasks_idler.cancel() @@ -528,6 +556,7 @@ class ThreadedFTPServer(_SpawnerBase): """A modified version of base FTPServer class which spawns a thread every time a new connection is established. """ + # The timeout passed to thread's IOLoop.poll() call on every # loop. Necessary since threads ignore KeyboardInterrupt. poll_timeout = 1.0 @@ -541,6 +570,7 @@ def _start_task(self, *args, **kwargs): if os.name == 'posix': try: import multiprocessing + multiprocessing.Lock() except Exception: # noqa # see https://github.com/giampaolo/pyftpdlib/issues/496 @@ -552,6 +582,7 @@ class MultiprocessFTPServer(_SpawnerBase): """A modified version of base FTPServer class which spawns a process every time a new connection is established. """ + _lock = multiprocessing.Lock() _exit = multiprocessing.Event() diff --git a/pyftpdlib/test/__init__.py b/pyftpdlib/test/__init__.py index 2d1f6e77..19f2e712 100644 --- a/pyftpdlib/test/__init__.py +++ b/pyftpdlib/test/__init__.py @@ -55,7 +55,7 @@ CI_TESTING = APPVEYOR or GITHUB_ACTIONS COVERAGE = 'COVERAGE_RUN' in os.environ # are we a 64 bit process? -IS_64BIT = sys.maxsize > 2 ** 32 +IS_64BIT = sys.maxsize > 2**32 OSX = sys.platform.startswith("darwin") POSIX = os.name == 'posix' WINDOWS = os.name == 'nt' @@ -94,13 +94,16 @@ def setUp(self): def tearDown(self): if not hasattr(self, "_test_ctx"): - raise AssertionError("super().setUp() was not called for this " - "test class") + raise AssertionError( + "super().setUp() was not called for this test class" + ) threads = set(threading.enumerate()) if len(threads) > len(self._test_ctx["threads"]): extra = threads - self._test_ctx["threads"] - raise AssertionError("%s orphaned thread(s) were left " - "behind: %r" % (len(extra), extra)) + raise AssertionError( + "%s orphaned thread(s) were left behind: %r" + % (len(extra), extra) + ) def __str__(self): # Print a full path representation of the single unit tests @@ -109,7 +112,10 @@ def __str__(self): if not fqmod.startswith('pyftpdlib.'): fqmod = 'pyftpdlib.test.' + fqmod return "%s.%s.%s" % ( - fqmod, self.__class__.__name__, self._testMethodName) + fqmod, + self.__class__.__name__, + self._testMethodName, + ) # assertRaisesRegexp renamed to assertRaisesRegex in 3.3; # add support for the new name. @@ -171,6 +177,7 @@ def get_testfn(suffix="", dir=None): def safe_rmpath(path): """Convenience function for removing temporary test files or dirs.""" + def retry_fun(fun): # On Windows it could happen that the file or directory has # open handles or references preventing the delete operation @@ -184,8 +191,9 @@ def retry_fun(fun): pass except WindowsError as _: err = _ - warnings.warn("ignoring %s" % str(err), UserWarning, - stacklevel=2) + warnings.warn( + "ignoring %s" % str(err), UserWarning, stacklevel=2 + ) time.sleep(0.01) raise err @@ -219,9 +227,9 @@ def configure_logging(): logger.addHandler(handler) - def disable_log_warning(fun): """Temporarily set FTP server's logging level to ERROR.""" + @functools.wraps(fun) def wrapper(self, *args, **kwargs): logger = logging.getLogger('pyftpdlib') @@ -231,6 +239,7 @@ def wrapper(self, *args, **kwargs): return fun(self, *args, **kwargs) finally: logger.setLevel(level) + return wrapper @@ -249,13 +258,14 @@ def cleanup(): class retry: """A retry decorator.""" - def __init__(self, - exception=Exception, - timeout=None, - retries=None, - interval=0.001, - logfun=None, - ): + def __init__( + self, + exception=Exception, + timeout=None, + retries=None, + interval=0.001, + logfun=None, + ): if timeout and retries: raise ValueError("timeout and retries args are mutually exclusive") self.exception = exception @@ -311,11 +321,13 @@ def retry_on_failure(retries=NO_RETRIES): """Decorator which runs a test function and retries N times before actually failing. """ + def logfun(exc): print("%r, retrying" % exc, file=sys.stderr) # NOQA - return retry(exception=AssertionError, timeout=None, retries=retries, - logfun=logfun) + return retry( + exception=AssertionError, timeout=None, retries=retries, logfun=logfun + ) def call_until(fun, expr, timeout=GLOBAL_TIMEOUT): @@ -344,6 +356,7 @@ def get_server_handler(): # commented out as per bug http://bugs.python.org/issue10354 # tempfile.template = 'tmp-pyftpdlib' + def setup_server(handler, server_class, addr=None): addr = (HOST, 0) if addr is None else addr authorizer = DummyAuthorizer() @@ -368,8 +381,11 @@ def assert_free_resources(parent_pid=None): this_proc = psutil.Process(parent_pid or os.getpid()) children = this_proc.children() if children: - warnings.warn("some children didn't terminate %r" % str(children), - UserWarning, stacklevel=2) + warnings.warn( + "some children didn't terminate %r" % str(children), + UserWarning, + stacklevel=2, + ) for child in children: try: child.kill() @@ -378,11 +394,17 @@ def assert_free_resources(parent_pid=None): pass # check unclosed connections if POSIX: - cons = [x for x in this_proc.connections('tcp') - if x.status != psutil.CONN_CLOSE_WAIT] + cons = [ + x + for x in this_proc.connections('tcp') + if x.status != psutil.CONN_CLOSE_WAIT + ] if cons: - warnings.warn("some connections didn't close %r" % str(cons), - UserWarning, stacklevel=2) + warnings.warn( + "some connections didn't close %r" % str(cons), + UserWarning, + stacklevel=2, + ) def reset_server_opts(): @@ -392,8 +414,9 @@ def reset_server_opts(): import pyftpdlib.servers # Control handlers. - tls_handler = getattr(pyftpdlib.handlers, "TLS_FTPHandler", - pyftpdlib.handlers.FTPHandler) + tls_handler = getattr( + pyftpdlib.handlers, "TLS_FTPHandler", pyftpdlib.handlers.FTPHandler + ) for klass in (pyftpdlib.handlers.FTPHandler, tls_handler): klass.auth_failed_timeout = 0.001 klass.authorizer = DummyAuthorizer() @@ -416,8 +439,9 @@ def reset_server_opts(): klass.tls_data_required = False # Data handlers. - tls_handler = getattr(pyftpdlib.handlers, "TLS_DTPHandler", - pyftpdlib.handlers.DTPHandler) + tls_handler = getattr( + pyftpdlib.handlers, "TLS_DTPHandler", pyftpdlib.handlers.DTPHandler + ) for klass in (pyftpdlib.handlers.DTPHandler, tls_handler): klass.timeout = 300 klass.ac_in_buffer_size = 4096 @@ -427,8 +451,7 @@ def reset_server_opts(): pyftpdlib.handlers.ThrottledDTPHandler.auto_sized_buffers = True # Acceptors. - ls = [pyftpdlib.servers.FTPServer, - pyftpdlib.servers.ThreadedFTPServer] + ls = [pyftpdlib.servers.FTPServer, pyftpdlib.servers.ThreadedFTPServer] if POSIX: ls.append(pyftpdlib.servers.MultiprocessFTPServer) for klass in ls: @@ -442,6 +465,7 @@ class ThreadedTestFTPd(threading.Thread): wraps the polling loop into a thread. The instance returned can be start()ed and stop()ped. """ + handler = FTPHandler server_class = FTPServer poll_interval = 0.001 if CI_TESTING else 0.000001 @@ -462,8 +486,9 @@ def run(self): try: while not self._stop_flag: with self.lock: - self.server.serve_forever(timeout=self.poll_interval, - blocking=False) + self.server.serve_forever( + timeout=self.poll_interval, blocking=False + ) finally: self._event_stop.set() @@ -477,15 +502,18 @@ def stop(self): if POSIX: + class MProcessTestFTPd(multiprocessing.Process): """Same as above but using a sub process instead.""" + handler = FTPHandler server_class = FTPServer def __init__(self, addr=None): super().__init__() self.server = setup_server( - self.handler, self.server_class, addr=addr) + self.handler, self.server_class, addr=addr + ) self.host, self.port = self.server.socket.getsockname()[:2] self._started = False @@ -501,6 +529,7 @@ def stop(self): self.join() reset_server_opts() assert_free_resources() + else: # Windows MProcessTestFTPd = ThreadedTestFTPd diff --git a/pyftpdlib/test/runner.py b/pyftpdlib/test/runner.py index 5292d9d3..8680b817 100644 --- a/pyftpdlib/test/runner.py +++ b/pyftpdlib/test/runner.py @@ -30,7 +30,9 @@ VERBOSITY = 2 FAILED_TESTS_FNAME = '.failed-tests.txt' HERE = os.path.abspath(os.path.dirname(__file__)) -loadTestsFromTestCase = unittest.defaultTestLoader.loadTestsFromTestCase # noqa +loadTestsFromTestCase = ( # noqa + unittest.defaultTestLoader.loadTestsFromTestCase +) def term_supports_colors(file=sys.stdout): # pragma: no cover @@ -38,6 +40,7 @@ def term_supports_colors(file=sys.stdout): # pragma: no cover return True try: import curses + assert file.isatty() curses.setupterm() assert curses.tigetnum("colors") > 0 @@ -51,7 +54,8 @@ def term_supports_colors(file=sys.stdout): # pragma: no cover def print_color( - s, color=None, bold=False, file=sys.stdout): # pragma: no cover + s, color=None, bold=False, file=sys.stdout +): # pragma: no cover """Print a colorized version of string.""" if not term_supports_colors(): print(s, file=file) # NOQA @@ -62,16 +66,19 @@ def print_color( DEFAULT_COLOR = 7 GetStdHandle = ctypes.windll.Kernel32.GetStdHandle - SetConsoleTextAttribute = \ + SetConsoleTextAttribute = ( ctypes.windll.Kernel32.SetConsoleTextAttribute + ) colors = dict(green=2, red=4, brown=6, yellow=6) colors[None] = DEFAULT_COLOR try: color = colors[color] except KeyError: - raise ValueError("invalid color %r; choose between %r" % ( - color, list(colors.keys()))) + raise ValueError( + "invalid color %r; choose between %r" + % (color, list(colors.keys())) + ) if bold and color <= 7: color += 8 @@ -80,25 +87,34 @@ def print_color( handle = GetStdHandle(handle_id) SetConsoleTextAttribute(handle, color) try: - print(s, file=file) # NOQA + print(s, file=file) # NOQA finally: SetConsoleTextAttribute(handle, DEFAULT_COLOR) - def hilite(s, color=None, bold=False): # pragma: no cover """Return an highlighted version of 'string'.""" if not term_supports_colors(): return s attr = [] - colors = dict(green='32', red='91', brown='33', yellow='93', blue='34', - violet='35', lightblue='36', grey='37', darkgrey='30') + colors = dict( + green='32', + red='91', + brown='33', + yellow='93', + blue='34', + violet='35', + lightblue='36', + grey='37', + darkgrey='30', + ) colors[None] = '29' try: color = colors[color] except KeyError: - raise ValueError("invalid color %r; choose between %s" % ( - list(colors.keys()))) + raise ValueError( + "invalid color %r; choose between %s" % (list(colors.keys())) + ) attr.append(color) if bold: attr.append('1') @@ -109,9 +125,11 @@ def import_module_by_path(path): name = os.path.splitext(os.path.basename(path))[0] if sys.version_info[0] < 3: import imp + return imp.load_source(name, path) else: import importlib.util + spec = importlib.util.spec_from_file_location(name, path) mod = importlib.util.module_from_spec(spec) spec.loader.exec_module(mod) @@ -133,10 +151,13 @@ class TestLoader: skip_files = [] def _get_testmods(self): - return [os.path.join(self.testdir, x) - for x in os.listdir(self.testdir) - if x.startswith('test_') and x.endswith('.py') and - x not in self.skip_files] + return [ + os.path.join(self.testdir, x) + for x in os.listdir(self.testdir) + if x.startswith('test_') + and x.endswith('.py') + and x not in self.skip_files + ] def _iter_testmod_classes(self): """Iterate over all test files in this directory and return @@ -146,8 +167,9 @@ def _iter_testmod_classes(self): mod = import_module_by_path(path) for name in dir(mod): obj = getattr(mod, name) - if isinstance(obj, type) and \ - issubclass(obj, unittest.TestCase): + if isinstance(obj, type) and issubclass( + obj, unittest.TestCase + ): yield obj def all(self): @@ -269,9 +291,12 @@ def run_from_name(name): def main(): usage = "python3 -m pyftpdlib.test [opts] [test-name]" parser = optparse.OptionParser(usage=usage, description="run unit tests") - parser.add_option("--last-failed", - action="store_true", default=False, - help="only run last failed tests") + parser.add_option( + "--last-failed", + action="store_true", + default=False, + help="only run last failed tests", + ) opts, args = parser.parse_args() if not opts.last_failed: diff --git a/pyftpdlib/test/test_authorizers.py b/pyftpdlib/test/test_authorizers.py index 27c6c3b5..4189b71d 100644 --- a/pyftpdlib/test/test_authorizers.py +++ b/pyftpdlib/test/test_authorizers.py @@ -26,6 +26,7 @@ if POSIX: import pwd + try: from pyftpdlib.authorizers import UnixAuthorizer except ImportError: @@ -72,8 +73,8 @@ def test_common_methods(self): auth.add_anonymous(HOME) # check credentials auth.validate_authentication(USER, PASSWD, None) - self.assertRaises(AuthenticationFailed, - auth.validate_authentication, USER, 'wrongpwd', None) + with self.assertRaises(AuthenticationFailed): + auth.validate_authentication(USER, 'wrongpwd', None) auth.validate_authentication('anonymous', 'foo', None) auth.validate_authentication('anonymous', '', None) # empty passwd # remove them @@ -82,71 +83,67 @@ def test_common_methods(self): # raise exc if user does not exists self.assertRaises(KeyError, auth.remove_user, USER) # raise exc if path does not exist - self.assertRaisesRegex(ValueError, - 'no such directory', - auth.add_user, USER, PASSWD, '?:\\') - self.assertRaisesRegex(ValueError, - 'no such directory', - auth.add_anonymous, '?:\\') + with self.assertRaisesRegex(ValueError, 'no such directory'): + auth.add_user(USER, PASSWD, '?:\\') + with self.assertRaisesRegex(ValueError, 'no such directory'): + auth.add_anonymous('?:\\') # raise exc if user already exists auth.add_user(USER, PASSWD, HOME) auth.add_anonymous(HOME) - self.assertRaisesRegex(ValueError, - 'user %r already exists' % USER, - auth.add_user, USER, PASSWD, HOME) - self.assertRaisesRegex(ValueError, - "user 'anonymous' already exists", - auth.add_anonymous, HOME) + with self.assertRaisesRegex( + ValueError, 'user %r already exists' % USER + ): + auth.add_user(USER, PASSWD, HOME) + with self.assertRaisesRegex( + ValueError, "user 'anonymous' already exists" + ): + auth.add_anonymous(HOME) auth.remove_user(USER) auth.remove_user('anonymous') # raise on wrong permission - self.assertRaisesRegex(ValueError, - "no such permission", - auth.add_user, USER, PASSWD, HOME, perm='?') - self.assertRaisesRegex(ValueError, - "no such permission", - auth.add_anonymous, HOME, perm='?') + with self.assertRaisesRegex(ValueError, "no such permission"): + auth.add_user(USER, PASSWD, HOME, perm='?') + with self.assertRaisesRegex(ValueError, "no such permission"): + auth.add_anonymous(HOME, perm='?') # expect warning on write permissions assigned to anonymous user for x in "adfmw": - self.assertRaisesRegex( - RuntimeWarning, - "write permissions assigned to anonymous user.", - auth.add_anonymous, HOME, perm=x) + with self.assertRaisesRegex( + RuntimeWarning, "write permissions assigned to anonymous user." + ): + auth.add_anonymous(HOME, perm=x) def test_override_perm_interface(self): auth = DummyAuthorizer() auth.add_user(USER, PASSWD, HOME, perm='elr') # raise exc if user does not exists - self.assertRaises(KeyError, auth.override_perm, USER + 'w', - HOME, 'elr') + with self.assertRaises(KeyError): + auth.override_perm(USER + 'w', HOME, 'elr') # raise exc if path does not exist or it's not a directory - self.assertRaisesRegex(ValueError, - 'no such directory', - auth.override_perm, USER, '?:\\', 'elr') - self.assertRaisesRegex(ValueError, - 'no such directory', - auth.override_perm, USER, self.tempfile, 'elr') + with self.assertRaisesRegex(ValueError, 'no such directory'): + auth.override_perm(USER, '?:\\', 'elr') + with self.assertRaisesRegex(ValueError, 'no such directory'): + auth.override_perm(USER, self.tempfile, 'elr') # raise on wrong permission - self.assertRaisesRegex(ValueError, - "no such permission", auth.override_perm, - USER, HOME, perm='?') + with self.assertRaisesRegex(ValueError, "no such permission"): + auth.override_perm(USER, HOME, perm='?') # expect warning on write permissions assigned to anonymous user auth.add_anonymous(HOME) for p in "adfmw": - self.assertRaisesRegex( - RuntimeWarning, - "write permissions assigned to anonymous user.", - auth.override_perm, 'anonymous', HOME, p) + with self.assertRaisesRegex( + RuntimeWarning, "write permissions assigned to anonymous user." + ): + auth.override_perm('anonymous', HOME, p) # raise on attempt to override home directory permissions - self.assertRaisesRegex(ValueError, - "can't override home directory permissions", - auth.override_perm, USER, HOME, perm='w') + with self.assertRaisesRegex( + ValueError, "can't override home directory permissions" + ): + auth.override_perm(USER, HOME, perm='w') # raise on attempt to override a path escaping home directory if os.path.dirname(HOME) != HOME: - self.assertRaisesRegex(ValueError, - "path escapes user home directory", - auth.override_perm, USER, - os.path.dirname(HOME), perm='w') + with self.assertRaisesRegex( + ValueError, "path escapes user home directory" + ): + auth.override_perm(USER, os.path.dirname(HOME), perm='w') # try to re-set an overridden permission auth.override_perm(USER, self.tempdir, perm='w') auth.override_perm(USER, self.tempdir, perm='wr') @@ -164,8 +161,9 @@ def test_override_perm_recursive_paths(self): self.assertEqual(auth.has_perm(USER, 'w', HOME + '@'), False) self.assertEqual(auth.has_perm(USER, 'w', self.tempdir + '@'), False) - path = os.path.join(self.tempdir + '@', - os.path.basename(self.tempfile)) + path = os.path.join( + self.tempdir + '@', os.path.basename(self.tempfile) + ) self.assertEqual(auth.has_perm(USER, 'w', path), False) # test case-sensitiveness if (os.name in ('nt', 'ce')) or (sys.platform == 'cygwin'): @@ -184,19 +182,22 @@ def test_override_perm_not_recursive_paths(self): self.assertEqual(auth.has_perm(USER, 'w', HOME + '@'), False) self.assertEqual(auth.has_perm(USER, 'w', self.tempdir + '@'), False) - path = os.path.join(self.tempdir + '@', - os.path.basename(self.tempfile)) + path = os.path.join( + self.tempdir + '@', os.path.basename(self.tempfile) + ) self.assertEqual(auth.has_perm(USER, 'w', path), False) # test case-sensitiveness if (os.name in ('nt', 'ce')) or (sys.platform == 'cygwin'): - self.assertEqual(auth.has_perm(USER, 'w', self.tempdir.upper()), - True) + self.assertEqual( + auth.has_perm(USER, 'w', self.tempdir.upper()), True + ) class _SharedAuthorizerTests: """Tests valid for both UnixAuthorizer and WindowsAuthorizer for those parts which share the same API. """ + authorizer_class = None # --- utils @@ -239,6 +240,7 @@ def assertRaisesWithMsg(self, excClass, msg, callableObj, *args, **kwargs): else: excName = str(excClass) raise self.failureException("%s not raised" % excName) + # --- /utils def test_get_home_dir(self): @@ -249,8 +251,7 @@ def test_get_home_dir(self): self.assertTrue(os.path.isdir(home)) if auth.has_user('nobody'): home = auth.get_home_dir('nobody') - self.assertRaises(AuthorizerError, - auth.get_home_dir, nonexistent_user) + self.assertRaises(AuthorizerError, auth.get_home_dir, nonexistent_user) def test_has_user(self): auth = self.authorizer_class() @@ -272,10 +273,18 @@ def test_validate_authentication(self): nonexistent_user = self.get_nonexistent_user() self.assertRaises( AuthenticationFailed, - auth.validate_authentication, current_user, 'wrongpasswd', None) + auth.validate_authentication, + current_user, + 'wrongpasswd', + None, + ) self.assertRaises( AuthenticationFailed, - auth.validate_authentication, nonexistent_user, 'bar', None) + auth.validate_authentication, + nonexistent_user, + 'bar', + None, + ) def test_impersonate_user(self): auth = self.authorizer_class() @@ -285,14 +294,23 @@ def test_impersonate_user(self): auth.impersonate_user(self.get_current_user(), '') self.assertRaises( AuthorizerError, - auth.impersonate_user, nonexistent_user, 'pwd') + auth.impersonate_user, + nonexistent_user, + 'pwd', + ) else: self.assertRaises( Win32ExtError, - auth.impersonate_user, nonexistent_user, 'pwd') + auth.impersonate_user, + nonexistent_user, + 'pwd', + ) self.assertRaises( Win32ExtError, - auth.impersonate_user, self.get_current_user(), '') + auth.impersonate_user, + self.get_current_user(), + '', + ) finally: auth.terminate_impersonation('') @@ -321,35 +339,51 @@ def test_error_options(self): self.assertRaisesWithMsg( AuthorizerError, "rejected_users and allowed_users options are mutually exclusive", - self.authorizer_class, allowed_users=['foo'], - rejected_users=['bar']) + self.authorizer_class, + allowed_users=['foo'], + rejected_users=['bar'], + ) self.assertRaisesWithMsg( AuthorizerError, 'invalid username "anonymous"', - self.authorizer_class, allowed_users=['anonymous']) + self.authorizer_class, + allowed_users=['anonymous'], + ) self.assertRaisesWithMsg( AuthorizerError, 'invalid username "anonymous"', - self.authorizer_class, rejected_users=['anonymous']) + self.authorizer_class, + rejected_users=['anonymous'], + ) self.assertRaisesWithMsg( AuthorizerError, 'unknown user %s' % wrong_user, - self.authorizer_class, allowed_users=[wrong_user]) - self.assertRaisesWithMsg(AuthorizerError, - 'unknown user %s' % wrong_user, - self.authorizer_class, - rejected_users=[wrong_user]) + self.authorizer_class, + allowed_users=[wrong_user], + ) + self.assertRaisesWithMsg( + AuthorizerError, + 'unknown user %s' % wrong_user, + self.authorizer_class, + rejected_users=[wrong_user], + ) def test_override_user_password(self): auth = self.authorizer_class() user = self.get_current_user() auth.override_user(user, password='foo') auth.validate_authentication(user, 'foo', None) - self.assertRaises(AuthenticationFailed, auth.validate_authentication, - user, 'bar', None) + self.assertRaises( + AuthenticationFailed, + auth.validate_authentication, + user, + 'bar', + None, + ) # make sure other settings keep using default values - self.assertEqual(auth.get_home_dir(user), - self.get_current_user_homedir()) + self.assertEqual( + auth.get_home_dir(user), self.get_current_user_homedir() + ) self.assertEqual(auth.get_perms(user), "elradfmwMT") self.assertEqual(auth.get_msg_login(user), "Login successful.") self.assertEqual(auth.get_msg_quit(user), "Goodbye.") @@ -373,8 +407,9 @@ def test_override_user_perm(self): auth.override_user(user, perm="elr") self.assertEqual(auth.get_perms(user), "elr") # make sure other settings keep using default values - self.assertEqual(auth.get_home_dir(user), - self.get_current_user_homedir()) + self.assertEqual( + auth.get_home_dir(user), self.get_current_user_homedir() + ) # self.assertEqual(auth.get_perms(user), "elradfmwMT") self.assertEqual(auth.get_msg_login(user), "Login successful.") self.assertEqual(auth.get_msg_quit(user), "Goodbye.") @@ -386,8 +421,9 @@ def test_override_user_msg_login_quit(self): self.assertEqual(auth.get_msg_login(user), "foo") self.assertEqual(auth.get_msg_quit(user), "bar") # make sure other settings keep using default values - self.assertEqual(auth.get_home_dir(user), - self.get_current_user_homedir()) + self.assertEqual( + auth.get_home_dir(user), self.get_current_user_homedir() + ) self.assertEqual(auth.get_perms(user), "elradfmwMT") # self.assertEqual(auth.get_msg_login(user), "Login successful.") # self.assertEqual(auth.get_msg_quit(user), "Goodbye.") @@ -406,33 +442,51 @@ def test_override_user_errors(self): self.assertRaisesWithMsg( AuthorizerError, "at least one keyword argument must be specified", - auth.override_user, this_user) - self.assertRaisesWithMsg(AuthorizerError, - 'no such user %s' % nonexistent_user, - auth.override_user, nonexistent_user, - perm='r') + auth.override_user, + this_user, + ) + self.assertRaisesWithMsg( + AuthorizerError, + 'no such user %s' % nonexistent_user, + auth.override_user, + nonexistent_user, + perm='r', + ) if self.authorizer_class.__name__ == 'UnixAuthorizer': - auth = self.authorizer_class(allowed_users=[this_user], - require_valid_shell=False) + auth = self.authorizer_class( + allowed_users=[this_user], require_valid_shell=False + ) else: auth = self.authorizer_class(allowed_users=[this_user]) auth.override_user(this_user, perm='r') - self.assertRaisesWithMsg(AuthorizerError, - '%s is not an allowed user' % another_user, - auth.override_user, another_user, perm='r') + self.assertRaisesWithMsg( + AuthorizerError, + '%s is not an allowed user' % another_user, + auth.override_user, + another_user, + perm='r', + ) if self.authorizer_class.__name__ == 'UnixAuthorizer': - auth = self.authorizer_class(rejected_users=[this_user], - require_valid_shell=False) + auth = self.authorizer_class( + rejected_users=[this_user], require_valid_shell=False + ) else: auth = self.authorizer_class(rejected_users=[this_user]) auth.override_user(another_user, perm='r') - self.assertRaisesWithMsg(AuthorizerError, - '%s is not an allowed user' % this_user, - auth.override_user, this_user, perm='r') - self.assertRaisesWithMsg(AuthorizerError, - "can't assign password to anonymous user", - auth.override_user, "anonymous", - password='foo') + self.assertRaisesWithMsg( + AuthorizerError, + '%s is not an allowed user' % this_user, + auth.override_user, + this_user, + perm='r', + ) + self.assertRaisesWithMsg( + AuthorizerError, + "can't assign password to anonymous user", + auth.override_user, + "anonymous", + password='foo', + ) # ===================================================================== @@ -441,8 +495,9 @@ def test_override_user_errors(self): @unittest.skipUnless(POSIX, "UNIX only") -@unittest.skipUnless(UnixAuthorizer is not None, - "UnixAuthorizer class not available") +@unittest.skipUnless( + UnixAuthorizer is not None, "UnixAuthorizer class not available" +) class TestUnixAuthorizer(_SharedAuthorizerTests, PyftpdlibTestCase): """Unix authorizer specific tests.""" @@ -457,7 +512,8 @@ def setUp(self): def test_get_perms_anonymous(self): auth = UnixAuthorizer( - global_perm='elr', anonymous_user=self.get_current_user()) + global_perm='elr', anonymous_user=self.get_current_user() + ) self.assertIn('e', auth.get_perms('anonymous')) self.assertNotIn('w', auth.get_perms('anonymous')) warnings.filterwarnings("ignore") @@ -467,7 +523,8 @@ def test_get_perms_anonymous(self): def test_has_perm_anonymous(self): auth = UnixAuthorizer( - global_perm='elr', anonymous_user=self.get_current_user()) + global_perm='elr', anonymous_user=self.get_current_user() + ) self.assertTrue(auth.has_perm(self.get_current_user(), 'r')) self.assertFalse(auth.has_perm(self.get_current_user(), 'w')) self.assertTrue(auth.has_perm('anonymous', 'e')) @@ -480,21 +537,41 @@ def test_has_perm_anonymous(self): def test_validate_authentication(self): # we can only test for invalid credentials auth = UnixAuthorizer(require_valid_shell=False) - self.assertRaises(AuthenticationFailed, - auth.validate_authentication, '?!foo', '?!foo', None) + self.assertRaises( + AuthenticationFailed, + auth.validate_authentication, + '?!foo', + '?!foo', + None, + ) auth = UnixAuthorizer(require_valid_shell=True) - self.assertRaises(AuthenticationFailed, - auth.validate_authentication, '?!foo', '?!foo', None) + self.assertRaises( + AuthenticationFailed, + auth.validate_authentication, + '?!foo', + '?!foo', + None, + ) def test_validate_authentication_anonymous(self): current_user = self.get_current_user() - auth = UnixAuthorizer(anonymous_user=current_user, - require_valid_shell=False) - self.assertRaises(AuthenticationFailed, - auth.validate_authentication, 'foo', 'passwd', None) + auth = UnixAuthorizer( + anonymous_user=current_user, require_valid_shell=False + ) + self.assertRaises( + AuthenticationFailed, + auth.validate_authentication, + 'foo', + 'passwd', + None, + ) self.assertRaises( AuthenticationFailed, - auth.validate_authentication, current_user, 'passwd', None) + auth.validate_authentication, + current_user, + 'passwd', + None, + ) auth.validate_authentication('anonymous', 'passwd', None) def test_require_valid_shell(self): @@ -513,7 +590,9 @@ def get_fake_shell_user(): self.assertRaisesWithMsg( AuthorizerError, "user %s has not a valid shell" % user, - UnixAuthorizer, allowed_users=[user]) + UnixAuthorizer, + allowed_users=[user], + ) # commented as it first fails for invalid home # self.assertRaisesWithMsg( # ValueError, @@ -522,18 +601,24 @@ def get_fake_shell_user(): auth = UnixAuthorizer() self.assertTrue(auth._has_valid_shell(self.get_current_user())) self.assertFalse(auth._has_valid_shell(user)) - self.assertRaisesWithMsg(AuthorizerError, - "User %s doesn't have a valid shell." % user, - auth.override_user, user, perm='r') + self.assertRaisesWithMsg( + AuthorizerError, + "User %s doesn't have a valid shell." % user, + auth.override_user, + user, + perm='r', + ) def test_not_root(self): # UnixAuthorizer is supposed to work only as super user auth = self.authorizer_class() try: auth.impersonate_user('nobody', '') - self.assertRaisesWithMsg(AuthorizerError, - "super user privileges are required", - UnixAuthorizer) + self.assertRaisesWithMsg( + AuthorizerError, + "super user privileges are required", + UnixAuthorizer, + ) finally: auth.terminate_impersonation('nobody') @@ -551,11 +636,13 @@ class TestWindowsAuthorizer(_SharedAuthorizerTests, PyftpdlibTestCase): def test_wrong_anonymous_credentials(self): user = self.get_current_user() - self.assertRaises(Win32ExtError, self.authorizer_class, - anonymous_user=user, - anonymous_password='$|1wrongpasswd') + with self.assertRaises(Win32ExtError): + self.authorizer_class( + anonymous_user=user, anonymous_password='$|1wrongpasswd' + ) if __name__ == '__main__': from pyftpdlib.test.runner import run_from_name + run_from_name(__file__) diff --git a/pyftpdlib/test/test_filesystems.py b/pyftpdlib/test/test_filesystems.py index 5a9bc591..acdfc884 100644 --- a/pyftpdlib/test/test_filesystems.py +++ b/pyftpdlib/test/test_filesystems.py @@ -209,4 +209,5 @@ def test_case(self): if __name__ == '__main__': from pyftpdlib.test.runner import run_from_name + run_from_name(__file__) diff --git a/pyftpdlib/test/test_functional.py b/pyftpdlib/test/test_functional.py index c39ae8d4..62f11479 100644 --- a/pyftpdlib/test/test_functional.py +++ b/pyftpdlib/test/test_functional.py @@ -67,6 +67,7 @@ class TestFtpAuthentication(PyftpdlibTestCase): """Test: USER, PASS, REIN.""" + server_class = MProcessTestFTPd client_class = ftplib.FTP @@ -90,8 +91,10 @@ def tearDown(self): super().tearDown() def assert_auth_failed(self, user, passwd): - self.assertRaisesRegex(ftplib.error_perm, '530 Authentication failed', - self.client.login, user, passwd) + with self.assertRaisesRegex( + ftplib.error_perm, '530 Authentication failed' + ): + self.client.login(user, passwd) def test_auth_ok(self): self.client.login(user=USER, passwd=PASSWD) @@ -113,12 +116,15 @@ def test_auth_failed(self): self.assert_auth_failed('wrong', 'wrong') def test_wrong_cmds_order(self): - self.assertRaisesRegex(ftplib.error_perm, '503 Login with USER first', - self.client.sendcmd, 'pass ' + PASSWD) + with self.assertRaisesRegex( + ftplib.error_perm, '503 Login with USER first' + ): + self.client.sendcmd('pass ' + PASSWD) self.client.login(user=USER, passwd=PASSWD) - self.assertRaisesRegex(ftplib.error_perm, - "503 User already authenticated.", - self.client.sendcmd, 'pass ' + PASSWD) + with self.assertRaisesRegex( + ftplib.error_perm, "503 User already authenticated." + ): + self.client.sendcmd('pass ' + PASSWD) def test_max_auth(self): self.assert_auth_failed(USER, 'wrong') @@ -129,16 +135,17 @@ def test_max_auth(self): # on the 'dead' socket object. If socket object is really # closed it should be raised a socket.error exception (Windows) # or a EOFError exception (Linux). - self.client.sock.settimeout(.1) + self.client.sock.settimeout(0.1) self.assertRaises((socket.error, EOFError), self.client.sendcmd, '') def test_rein(self): self.client.login(user=USER, passwd=PASSWD) self.client.sendcmd('rein') # user not authenticated, error response expected - self.assertRaisesRegex(ftplib.error_perm, - '530 Log in with USER and PASS first', - self.client.sendcmd, 'pwd') + with self.assertRaisesRegex( + ftplib.error_perm, '530 Log in with USER and PASS first' + ): + self.client.sendcmd('pwd') # by logging-in again we should be able to execute a # file-system command self.client.login(user=USER, passwd=PASSWD) @@ -167,17 +174,19 @@ def test_rein_during_transfer(self): rein_sent = True # flush account, error response expected self.client.sendcmd('rein') - self.assertRaisesRegex( + with self.assertRaisesRegex( ftplib.error_perm, '530 Log in with USER and PASS first', - self.client.dir) + ): + self.client.dir() # a 226 response is expected once transfer finishes self.assertEqual(self.client.voidresp()[:3], '226') # account is still flushed, error response is still expected - self.assertRaisesRegex(ftplib.error_perm, - '530 Log in with USER and PASS first', - self.client.sendcmd, 'size ' + self.testfn) + with self.assertRaisesRegex( + ftplib.error_perm, '530 Log in with USER and PASS first' + ): + self.client.sendcmd('size ' + self.testfn) # by logging-in again we should be able to execute a # filesystem command self.client.login(user=USER, passwd=PASSWD) @@ -192,9 +201,10 @@ def test_user(self): # is in progress. self.client.login(user=USER, passwd=PASSWD) self.client.sendcmd('user ' + USER) # authentication flushed - self.assertRaisesRegex(ftplib.error_perm, - '530 Log in with USER and PASS first', - self.client.sendcmd, 'pwd') + with self.assertRaisesRegex( + ftplib.error_perm, '530 Log in with USER and PASS first' + ): + self.client.sendcmd('pwd') self.client.sendcmd('pass ' + PASSWD) self.client.sendcmd('pwd') @@ -221,17 +231,19 @@ def test_user_during_transfer(self): rein_sent = True # flush account, expect an error response self.client.sendcmd('user ' + USER) - self.assertRaisesRegex( + with self.assertRaisesRegex( ftplib.error_perm, '530 Log in with USER and PASS first', - self.client.dir) + ): + self.client.dir() # a 226 response is expected once transfer finishes self.assertEqual(self.client.voidresp()[:3], '226') # account is still flushed, error response is still expected - self.assertRaisesRegex(ftplib.error_perm, - '530 Log in with USER and PASS first', - self.client.sendcmd, 'pwd') + with self.assertRaisesRegex( + ftplib.error_perm, '530 Log in with USER and PASS first' + ): + self.client.sendcmd('pwd') # by logging-in again we should be able to execute a # filesystem command self.client.sendcmd('pass ' + PASSWD) @@ -244,6 +256,7 @@ def test_user_during_transfer(self): class TestFtpDummyCmds(PyftpdlibTestCase): """Test: TYPE, STRU, MODE, NOOP, SYST, ALLO, HELP, SITE HELP.""" + server_class = MProcessTestFTPd client_class = ftplib.FTP @@ -302,16 +315,19 @@ def test_help(self): def test_site(self): self.assertRaises(ftplib.error_perm, self.client.sendcmd, 'site') self.assertRaises(ftplib.error_perm, self.client.sendcmd, 'site ?!?') - self.assertRaises(ftplib.error_perm, self.client.sendcmd, - 'site foo bar') - self.assertRaises(ftplib.error_perm, self.client.sendcmd, - 'sitefoo bar') + self.assertRaises( + ftplib.error_perm, self.client.sendcmd, 'site foo bar' + ) + self.assertRaises( + ftplib.error_perm, self.client.sendcmd, 'sitefoo bar' + ) def test_site_help(self): self.client.sendcmd('site help') self.client.sendcmd('site help help') - self.assertRaises(ftplib.error_perm, self.client.sendcmd, - 'site help ?!?') + self.assertRaises( + ftplib.error_perm, self.client.sendcmd, 'site help ?!?' + ) def test_rest(self): # Test error conditions only; resumed data transfers are @@ -323,8 +339,10 @@ def test_rest(self): self.assertRaises(ftplib.error_perm, self.client.sendcmd, 'rest 10.1') # REST is not supposed to be allowed in ASCII mode self.client.sendcmd('type a') - self.assertRaisesRegex(ftplib.error_perm, 'not allowed in ASCII mode', - self.client.sendcmd, 'rest 10') + with self.assertRaisesRegex( + ftplib.error_perm, 'not allowed in ASCII mode' + ): + self.client.sendcmd('rest 10') def test_feat(self): resp = self.client.sendcmd('feat') @@ -332,45 +350,73 @@ def test_feat(self): self.assertIn('TVFS', resp) def test_opts_feat(self): - self.assertRaises( - ftplib.error_perm, self.client.sendcmd, 'opts mlst bad_fact') - self.assertRaises( - ftplib.error_perm, self.client.sendcmd, 'opts mlst type ;') - self.assertRaises(ftplib.error_perm, self.client.sendcmd, - 'opts not_mlst') + with self.assertRaises(ftplib.error_perm): + self.client.sendcmd('opts mlst bad_fact') + with self.assertRaises(ftplib.error_perm): + self.client.sendcmd('opts mlst type ;') + with self.assertRaises(ftplib.error_perm): + self.client.sendcmd('opts not_mlst') # utility function which used for extracting the MLST "facts" # string from the FEAT response def mlst(): resp = self.client.sendcmd('feat') return re.search(r'^\s*MLST\s+(\S+)$', resp, re.MULTILINE).group(1) + # we rely on "type", "perm", "size", and "modify" facts which # are those available on all platforms self.assertIn('type*;perm*;size*;modify*;', mlst()) - self.assertEqual(self.client.sendcmd( - 'opts mlst type;'), '200 MLST OPTS type;') - self.assertEqual(self.client.sendcmd( - 'opts mLSt TypE;'), '200 MLST OPTS type;') + self.assertEqual( + self.client.sendcmd('opts mlst type;'), '200 MLST OPTS type;' + ) + self.assertEqual( + self.client.sendcmd('opts mLSt TypE;'), '200 MLST OPTS type;' + ) self.assertIn('type*;perm;size;modify;', mlst()) self.assertEqual(self.client.sendcmd('opts mlst'), '200 MLST OPTS ') self.assertNotIn('*', mlst()) self.assertEqual( - self.client.sendcmd('opts mlst fish;cakes;'), '200 MLST OPTS ') + self.client.sendcmd('opts mlst fish;cakes;'), '200 MLST OPTS ' + ) self.assertNotIn('*', mlst()) - self.assertEqual(self.client.sendcmd('opts mlst fish;cakes;type;'), - '200 MLST OPTS type;') + self.assertEqual( + self.client.sendcmd('opts mlst fish;cakes;type;'), + '200 MLST OPTS type;', + ) self.assertIn('type*;perm;size;modify;', mlst()) class TestFtpCmdsSemantic(PyftpdlibTestCase): server_class = MProcessTestFTPd client_class = ftplib.FTP - arg_cmds = \ - ['allo', 'appe', 'dele', 'eprt', 'mdtm', 'mfmt', 'mode', 'mkd', 'opts', - 'port', 'rest', 'retr', 'rmd', 'rnfr', 'rnto', 'site', 'size', 'stor', - 'stru', 'type', 'user', 'xmkd', 'xrmd', 'site chmod'] + arg_cmds = [ + 'allo', + 'appe', + 'dele', + 'eprt', + 'mdtm', + 'mfmt', + 'mkd', + 'mode', + 'opts', + 'port', + 'rest', + 'retr', + 'rmd', + 'rnfr', + 'rnto', + 'site chmod', + 'site', + 'size', + 'stor', + 'stru', + 'type', + 'user', + 'xmkd', + 'xrmd', + ] def setUp(self): super().setUp() @@ -396,8 +442,19 @@ def test_arg_cmds(self): def test_no_arg_cmds(self): # Test commands accepting no arguments. expected = "501 Syntax error: command does not accept arguments." - narg_cmds = ['abor', 'cdup', 'feat', 'noop', 'pasv', 'pwd', 'quit', - 'rein', 'syst', 'xcup', 'xpwd'] + narg_cmds = [ + 'abor', + 'cdup', + 'feat', + 'noop', + 'pasv', + 'pwd', + 'quit', + 'rein', + 'syst', + 'xcup', + 'xpwd', + ] for cmd in narg_cmds: self.client.putcmd(cmd + ' arg') resp = self.client.getmultiline() @@ -409,9 +466,22 @@ def test_auth_cmds(self): self.client.sendcmd('rein') for cmd in self.server.handler.proto_cmds: cmd = cmd.lower() - if cmd in ('feat', 'help', 'noop', 'user', 'pass', 'stat', 'syst', - 'quit', 'site', 'site help', 'pbsz', 'auth', 'prot', - 'ccc'): + if cmd in ( + 'feat', + 'help', + 'noop', + 'user', + 'pass', + 'stat', + 'syst', + 'quit', + 'site', + 'site help', + 'pbsz', + 'auth', + 'prot', + 'ccc', + ): continue if cmd in self.arg_cmds: cmd = cmd + ' arg' @@ -426,8 +496,8 @@ def test_no_auth_cmds(self): self.client.sendcmd(cmd) # STAT provided with an argument is equal to LIST hence not allowed # if not authenticated - self.assertRaisesRegex(ftplib.error_perm, '530 Log in with USER', - self.client.sendcmd, 'stat /') + with self.assertRaisesRegex(ftplib.error_perm, '530 Log in with USER'): + self.client.sendcmd('stat /') self.client.sendcmd('quit') @@ -435,6 +505,7 @@ class TestFtpFsOperations(PyftpdlibTestCase): """Test: PWD, CWD, CDUP, SIZE, RNFR, RNTO, DELE, MKD, RMD, MDTM, STAT, MFMT. """ + server_class = MProcessTestFTPd client_class = ftplib.FTP @@ -476,8 +547,9 @@ def test_cdup(self): self.client.cwd(self.tempdir) self.assertEqual(self.client.pwd(), '/%s' % self.tempdir) self.client.cwd(subfolder) - self.assertEqual(self.client.pwd(), - '/%s/%s' % (self.tempdir, subfolder)) + self.assertEqual( + self.client.pwd(), '/%s/%s' % (self.tempdir, subfolder) + ) self.client.sendcmd('cdup') self.assertEqual(self.client.pwd(), '/%s' % self.tempdir) self.client.sendcmd('cdup') @@ -507,9 +579,10 @@ def test_rmd(self): self.client.rmd(self.tempdir) self.assertRaises(ftplib.error_perm, self.client.rmd, self.tempfile) # make sure we can't remove the root directory - self.assertRaisesRegex(ftplib.error_perm, - "Can't remove root directory", - self.client.rmd, u('/')) + with self.assertRaisesRegex( + ftplib.error_perm, "Can't remove root directory" + ): + self.client.rmd(u('/')) def test_dele(self): self.client.delete(self.tempfile) @@ -526,23 +599,25 @@ def test_rnfr_rnto(self): self.client.rename(tempname, self.tempdir) # rnfr/rnto over non-existing paths bogus = self.get_testfn() - self.assertRaises(ftplib.error_perm, self.client.rename, bogus, '/x') - self.assertRaises( - ftplib.error_perm, self.client.rename, self.tempfile, u('/')) + with self.assertRaises(ftplib.error_perm): + self.client.rename(bogus, '/x') + with self.assertRaises(ftplib.error_perm): + self.client.rename(self.tempfile, u('/')) # rnto sent without first specifying the source - self.assertRaises(ftplib.error_perm, self.client.sendcmd, - 'rnto ' + self.tempfile) + with self.assertRaises(ftplib.error_perm): + self.client.sendcmd('rnto ' + self.tempfile) # make sure we can't rename root directory - self.assertRaisesRegex(ftplib.error_perm, - "Can't rename home directory", - self.client.rename, '/', '/x') + with self.assertRaisesRegex( + ftplib.error_perm, "Can't rename home directory" + ): + self.client.rename('/', '/x') def test_mdtm(self): self.client.sendcmd('mdtm ' + self.tempfile) bogus = self.get_testfn() - self.assertRaises(ftplib.error_perm, self.client.sendcmd, - 'mdtm ' + bogus) + with self.assertRaises(ftplib.error_perm): + self.client.sendcmd('mdtm ' + bogus) # make sure we can't use mdtm against directories try: self.client.sendcmd('mdtm ' + self.tempdir) @@ -566,7 +641,8 @@ def test_invalid_mfmt_timeval(self): try: self.client.sendcmd( - 'mfmt ' + test_timestamp_with_chars + ' ' + self.tempfile) + 'mfmt ' + test_timestamp_with_chars + ' ' + self.tempfile + ) except ftplib.error_perm as err: self.assertIn('Invalid time format', str(err)) else: @@ -574,7 +650,8 @@ def test_invalid_mfmt_timeval(self): try: self.client.sendcmd( - 'mfmt ' + test_timestamp_invalid_length + ' ' + self.tempfile) + 'mfmt ' + test_timestamp_invalid_length + ' ' + self.tempfile + ) except ftplib.error_perm as err: self.assertIn('Invalid time format', str(err)) else: @@ -603,21 +680,24 @@ def test_size(self): self.fail('Exception not raised') if not hasattr(os, 'chmod'): + def test_site_chmod(self): - self.assertRaises(ftplib.error_perm, self.client.sendcmd, - 'site chmod 777 ' + self.tempfile) + with self.assertRaises(ftplib.error_perm): + self.client.sendcmd('site chmod 777 ' + self.tempfile) + else: + def test_site_chmod(self): # not enough args - self.assertRaises(ftplib.error_perm, - self.client.sendcmd, 'site chmod 777') + with self.assertRaises(ftplib.error_perm): + self.client.sendcmd('site chmod 777') # bad args - self.assertRaises(ftplib.error_perm, self.client.sendcmd, - 'site chmod -177 ' + self.tempfile) - self.assertRaises(ftplib.error_perm, self.client.sendcmd, - 'site chmod 778 ' + self.tempfile) - self.assertRaises(ftplib.error_perm, self.client.sendcmd, - 'site chmod foo ' + self.tempfile) + with self.assertRaises(ftplib.error_perm): + self.client.sendcmd('site chmod -177 ' + self.tempfile) + with self.assertRaises(ftplib.error_perm): + self.client.sendcmd('site chmod 778 ' + self.tempfile) + with self.assertRaises(ftplib.error_perm): + self.client.sendcmd('site chmod foo ' + self.tempfile) def getmode(): mode = oct(stat.S_IMODE(os.stat(self.tempfile).st_mode)) @@ -660,6 +740,7 @@ def write(self, b): class TestFtpStoreData(PyftpdlibTestCase): """Test STOR, STOU, APPE, REST, TYPE.""" + server_class = MProcessTestFTPd client_class = ftplib.FTP use_sendfile = None @@ -696,8 +777,9 @@ def test_stor(self): self.dummy_sendfile.write(data) self.dummy_sendfile.seek(0) self.client.storbinary('stor ' + self.testfn, self.dummy_sendfile) - self.client.retrbinary('retr ' + self.testfn, - self.dummy_recvfile.write) + self.client.retrbinary( + 'retr ' + self.testfn, self.dummy_recvfile.write + ) self.dummy_recvfile.seek(0) datafile = self.dummy_recvfile.read() self.assertEqual(len(data), len(datafile)) @@ -730,8 +812,9 @@ def store(cmd, fp, blocksize=8192): self.dummy_sendfile.write(data) self.dummy_sendfile.seek(0) store('stor ' + self.testfn, self.dummy_sendfile) - self.client.retrbinary('retr ' + self.testfn, - self.dummy_recvfile.write) + self.client.retrbinary( + 'retr ' + self.testfn, self.dummy_recvfile.write + ) expected = data.replace(b'\r\n', b(os.linesep)) self.dummy_recvfile.seek(0) datafile = self.dummy_recvfile.read() @@ -767,8 +850,9 @@ def store(cmd, fp, blocksize=8192): store('stor ' + self.testfn, self.dummy_sendfile) expected = data.replace(b'\r\n', b(os.linesep)) - self.client.retrbinary('retr ' + self.testfn, - self.dummy_recvfile.write) + self.client.retrbinary( + 'retr ' + self.testfn, self.dummy_recvfile.write + ) self.client.quit() self.dummy_recvfile.seek(0) self.assertEqual(expected, self.dummy_recvfile.read()) @@ -797,8 +881,9 @@ def test_stou(self): conn.sendall(buf) # transfer finished, a 226 response is expected self.assertEqual('226', self.client.voidresp()[:3]) - self.client.retrbinary('retr ' + filename, - self.dummy_recvfile.write) + self.client.retrbinary( + 'retr ' + filename, self.dummy_recvfile.write + ) self.dummy_recvfile.seek(0) datafile = self.dummy_recvfile.read() self.assertEqual(len(data), len(datafile)) @@ -817,8 +902,10 @@ def test_stou_rest(self): # Watch for STOU preceded by REST, which makes no sense. self.client.sendcmd('type i') self.client.sendcmd('rest 10') - self.assertRaisesRegex(ftplib.error_temp, "Can't STOU while REST", - self.client.sendcmd, 'stou') + with self.assertRaisesRegex( + ftplib.error_temp, "Can't STOU while REST" + ): + self.client.sendcmd('stou') def test_stou_orphaned_file(self): # Check that no orphaned file gets left behind when STOU fails. @@ -831,8 +918,9 @@ def test_stou_orphaned_file(self): # login as a limited user in order to make STOU fail self.client.login('anonymous', '@nopasswd') before = os.listdir(HOME) - self.assertRaises(ftplib.error_perm, self.client.sendcmd, - 'stou ' + self.testfn) + self.assertRaises( + ftplib.error_perm, self.client.sendcmd, 'stou ' + self.testfn + ) after = os.listdir(HOME) if before != after: for file in after: @@ -850,7 +938,8 @@ def test_appe(self): self.client.storbinary('appe ' + self.testfn, self.dummy_sendfile) self.client.retrbinary( - "retr " + self.testfn, self.dummy_recvfile.write) + "retr " + self.testfn, self.dummy_recvfile.write + ) self.dummy_recvfile.seek(0) datafile = self.dummy_recvfile.read() self.assertEqual(len(data1 + data2), len(datafile)) @@ -860,8 +949,10 @@ def test_appe_rest(self): # Watch for APPE preceded by REST, which makes no sense. self.client.sendcmd('type i') self.client.sendcmd('rest 10') - self.assertRaisesRegex(ftplib.error_temp, "Can't APPE while REST", - self.client.sendcmd, 'appe x') + with self.assertRaisesRegex( + ftplib.error_temp, "Can't APPE while REST" + ): + self.client.sendcmd('appe x') def test_rest_on_stor(self): # Test STOR preceded by REST. @@ -871,7 +962,8 @@ def test_rest_on_stor(self): self.client.voidcmd('TYPE I') with contextlib.closing( - self.client.transfercmd('stor ' + self.testfn)) as conn: + self.client.transfercmd('stor ' + self.testfn) + ) as conn: bytes_sent = 0 while True: chunk = self.dummy_sendfile.read(BUFSIZE) @@ -892,13 +984,14 @@ def test_rest_on_stor(self): file_size = self.client.size(self.testfn) self.assertEqual(file_size, bytes_sent) self.client.sendcmd('rest %s' % (file_size + 1)) - self.assertRaises(ftplib.error_perm, self.client.sendcmd, - 'stor ' + self.testfn) + with self.assertRaises(ftplib.error_perm): + self.client.sendcmd('stor ' + self.testfn) self.client.sendcmd('rest %s' % bytes_sent) self.client.storbinary('stor ' + self.testfn, self.dummy_sendfile) - self.client.retrbinary('retr ' + self.testfn, - self.dummy_recvfile.write) + self.client.retrbinary( + 'retr ' + self.testfn, self.dummy_recvfile.write + ) self.dummy_sendfile.seek(0) self.dummy_recvfile.seek(0) @@ -911,8 +1004,8 @@ def test_failing_rest_on_stor(self): # Test REST -> STOR against a non existing file. self.client.sendcmd('type i') self.client.sendcmd('rest 10') - self.assertRaises(ftplib.error_perm, self.client.storbinary, - 'stor ' + self.testfn, lambda x: x) + with self.assertRaises(ftplib.error_perm): + self.client.storbinary('stor ' + self.testfn, lambda x: x) # if the first STOR failed because of REST, the REST marker # is supposed to be resetted to 0 self.dummy_sendfile.write(b'x' * 4096) @@ -924,7 +1017,8 @@ def test_quit_during_transfer(self): # progress, the connection must remain open for result response # and the server will then close it. with contextlib.closing( - self.client.transfercmd('stor ' + self.testfn)) as conn: + self.client.transfercmd('stor ' + self.testfn) + ) as conn: conn.sendall(b'abcde12345' * 50000) self.client.sendcmd('quit') conn.sendall(b'abcde12345' * 50000) @@ -933,9 +1027,10 @@ def test_quit_during_transfer(self): # Make sure client has been disconnected. # socket.error (Windows) or EOFError (Linux) exception is supposed # to be raised in such a case. - self.client.sock.settimeout(.1) - self.assertRaises((socket.error, EOFError), - self.client.sendcmd, 'noop') + self.client.sock.settimeout(0.1) + self.assertRaises( + (socket.error, EOFError), self.client.sendcmd, 'noop' + ) def test_stor_empty_file(self): self.client.storbinary('stor ' + self.testfn, self.dummy_sendfile) @@ -945,20 +1040,22 @@ def test_stor_empty_file(self): @unittest.skipUnless(POSIX, "POSIX only") -@unittest.skipIf(not PY3 and sendfile is None, - "pysendfile not installed") +@unittest.skipIf(not PY3 and sendfile is None, "pysendfile not installed") class TestFtpStoreDataNoSendfile(TestFtpStoreData): """Test STOR, STOU, APPE, REST, TYPE not using sendfile().""" + use_sendfile = False class TestFtpStoreDataWithCustomIO(TestFtpStoreData): """Test STOR, STOU, APPE, REST, TYPE with custom IO objects().""" + use_custom_io = True class TestFtpRetrieveData(PyftpdlibTestCase): """Test RETR, REST, TYPE.""" + server_class = MProcessTestFTPd client_class = ftplib.FTP use_sendfile = None @@ -967,8 +1064,7 @@ class TestFtpRetrieveData(PyftpdlibTestCase): def retrieve_ascii(self, cmd, callback, blocksize=8192, rest=None): """Like retrbinary but uses TYPE A instead.""" self.client.voidcmd('type a') - with contextlib.closing( - self.client.transfercmd(cmd, rest)) as conn: + with contextlib.closing(self.client.transfercmd(cmd, rest)) as conn: conn.settimeout(GLOBAL_TIMEOUT) while True: data = conn.recv(blocksize) @@ -1012,8 +1108,8 @@ def test_retr(self): # attempt to retrieve a file which doesn't exist bogus = self.get_testfn() - self.assertRaises(ftplib.error_perm, self.client.retrbinary, - "retr " + bogus, lambda x: x) + with self.assertRaises(ftplib.error_perm): + self.client.retrbinary("retr " + bogus, lambda x: x) def test_retr_ascii(self): # Test RETR in ASCII mode. @@ -1047,7 +1143,8 @@ def test_restore_on_retr(self): received_bytes = 0 self.client.voidcmd('TYPE I') with contextlib.closing( - self.client.transfercmd('retr ' + self.testfn)) as conn: + self.client.transfercmd('retr ' + self.testfn) + ) as conn: conn.settimeout(GLOBAL_TIMEOUT) while True: chunk = conn.recv(BUFSIZE) @@ -1066,8 +1163,9 @@ def test_restore_on_retr(self): # on retr (RFC-1123) file_size = self.client.size(self.testfn) self.client.sendcmd('rest %s' % (file_size + 1)) - self.assertRaises(ftplib.error_perm, self.client.sendcmd, - 'retr ' + self.testfn) + self.assertRaises( + ftplib.error_perm, self.client.sendcmd, 'retr ' + self.testfn + ) # test resume self.client.sendcmd('rest %s' % received_bytes) self.client.retrbinary("retr " + self.testfn, self.dummyfile.write) @@ -1084,20 +1182,22 @@ def test_retr_empty_file(self): @unittest.skipUnless(POSIX, "POSIX only") -@unittest.skipIf(not PY3 and sendfile is None, - "pysendfile not installed") +@unittest.skipIf(not PY3 and sendfile is None, "pysendfile not installed") class TestFtpRetrieveDataNoSendfile(TestFtpRetrieveData): """Test RETR, REST, TYPE by not using sendfile().""" + use_sendfile = False class TestFtpRetrieveDataCustomIO(TestFtpRetrieveData): """Test RETR, REST, TYPE using custom IO objects.""" + use_custom_io = True class TestFtpListingCmds(PyftpdlibTestCase): """Test LIST, NLST, argumented STAT.""" + server_class = MProcessTestFTPd client_class = ftplib.FTP @@ -1131,8 +1231,8 @@ def _test_listing_cmds(self, cmd): self.assertTrue(''.join(x).endswith(self.testfn)) # non-existent path, 550 response is expected bogus = self.get_testfn() - self.assertRaises(ftplib.error_perm, self.client.retrlines, - '%s ' % cmd + bogus, lambda x: x) + with self.assertRaises(ftplib.error_perm): + self.client.retrlines('%s ' % cmd + bogus, lambda x: x) # for an empty directory we excpect that the data channel is # opened anyway and that no data is received x = [] @@ -1172,14 +1272,16 @@ def mlstline(cmd): self.assertTrue(mlstline('mlst').startswith(' ')) # where TVFS is supported, a fully qualified pathname is expected self.assertTrue( - mlstline('mlst ' + self.testfn).endswith('/' + self.testfn)) + mlstline('mlst ' + self.testfn).endswith('/' + self.testfn) + ) self.assertTrue(mlstline('mlst').endswith('/')) # assume that no argument has the same meaning of "/" self.assertEqual(mlstline('mlst'), mlstline('mlst /')) # non-existent path bogus = self.get_testfn() - self.assertRaises(ftplib.error_perm, self.client.sendcmd, - 'mlst ' + bogus) + self.assertRaises( + ftplib.error_perm, self.client.sendcmd, 'mlst ' + bogus + ) # test file/dir notations self.assertIn('type=dir', mlstline('mlst')) self.assertIn('type=file', mlstline('mlst ' + self.testfn)) @@ -1238,8 +1340,9 @@ def test_stat(self): resp = self.client.getmultiline() self.assertEqual(resp, '550 Globbing not supported.') bogus = self.get_testfn() - self.assertRaises(ftplib.error_perm, self.client.sendcmd, - 'stat ' + bogus) + self.assertRaises( + ftplib.error_perm, self.client.sendcmd, 'stat ' + bogus + ) def test_unforeseen_time_event(self): # Emulate a case where the file last modification time is prior @@ -1263,6 +1366,7 @@ def test_unforeseen_time_event(self): class TestFtpAbort(PyftpdlibTestCase): """Test: ABOR.""" + server_class = MProcessTestFTPd client_class = ftplib.FTP @@ -1314,7 +1418,8 @@ def test_abor_during_transfer(self): f.write(data) self.client.voidcmd('TYPE I') with contextlib.closing( - self.client.transfercmd('retr ' + testfn)) as conn: + self.client.transfercmd('retr ' + testfn) + ) as conn: bytes_recv = 0 while bytes_recv < 65536: chunk = conn.recv(BUFSIZE) @@ -1347,6 +1452,7 @@ def test_oob_abor(self): class TestThrottleBandwidth(PyftpdlibTestCase): """Test ThrottledDTPHandler class.""" + server_class = MProcessTestFTPd client_class = ftplib.FTP @@ -1360,8 +1466,10 @@ class CustomDTPHandler(ThrottledDTPHandler): def _throttle_bandwidth(self, *args, **kwargs): ThrottledDTPHandler._throttle_bandwidth(self, *args, **kwargs) - if (self._throttler is not None and not - self._throttler.cancelled): + if ( + self._throttler is not None + and not self._throttler.cancelled + ): self._throttler.call() self._throttler = None @@ -1418,6 +1526,7 @@ class TestTimeouts(PyftpdlibTestCase): """Test idle-timeout capabilities of control and data channels. Some tests may fail on slow machines. """ + server_class = MProcessTestFTPd client_class = ftplib.FTP @@ -1427,8 +1536,13 @@ def setUp(self): self.client = None self.testfn = self.get_testfn() - def _setUp(self, idle_timeout=300, data_timeout=300, pasv_timeout=30, - port_timeout=30): + def _setUp( + self, + idle_timeout=300, + data_timeout=300, + pasv_timeout=30, + port_timeout=30, + ): self.server = self.server_class() self.server.handler.timeout = idle_timeout self.server.handler.dtp_handler.timeout = data_timeout @@ -1478,8 +1592,9 @@ def test_data_timeout(self): data = self.client.sock.recv(BUFSIZE) self.assertEqual(data, b"421 Data connection timed out.\r\n") # ensure client has been kicked off - self.assertRaises((socket.error, EOFError), self.client.sendcmd, - 'noop') + self.assertRaises( + (socket.error, EOFError), self.client.sendcmd, 'noop' + ) def test_data_timeout_not_reached(self): # Impose a timeout for the data channel, then keep sending data for a @@ -1487,7 +1602,8 @@ def test_data_timeout_not_reached(self): # whether the transfer stalled for with no progress is executed. self._setUp(data_timeout=0.5 if CI_TESTING else 0.1) with contextlib.closing( - self.client.transfercmd('stor ' + self.testfn)) as sock: + self.client.transfercmd('stor ' + self.testfn) + ) as sock: if hasattr(self.client_class, 'ssl_version'): sock = ssl.wrap_socket(sock) stop_at = time.time() + 0.2 @@ -1499,8 +1615,10 @@ def test_data_timeout_not_reached(self): def test_idle_data_timeout1(self): # Tests that the control connection timeout is suspended while # the data channel is opened - self._setUp(idle_timeout=0.5 if CI_TESTING else 0.1, - data_timeout=0.6 if CI_TESTING else 0.2) + self._setUp( + idle_timeout=0.5 if CI_TESTING else 0.1, + data_timeout=0.6 if CI_TESTING else 0.2, + ) addr = self.client.makepasv() with contextlib.closing(socket.socket()) as s: s.settimeout(GLOBAL_TIMEOUT) @@ -1510,14 +1628,17 @@ def test_idle_data_timeout1(self): data = self.client.sock.recv(BUFSIZE) self.assertEqual(data, b"421 Data connection timed out.\r\n") # ensure client has been kicked off - self.assertRaises((socket.error, EOFError), self.client.sendcmd, - 'noop') + self.assertRaises( + (socket.error, EOFError), self.client.sendcmd, 'noop' + ) def test_idle_data_timeout2(self): # Tests that the control connection timeout is restarted after # data channel has been closed - self._setUp(idle_timeout=0.5 if CI_TESTING else 0.1, - data_timeout=0.6 if CI_TESTING else 0.2) + self._setUp( + idle_timeout=0.5 if CI_TESTING else 0.1, + data_timeout=0.6 if CI_TESTING else 0.2, + ) addr = self.client.makepasv() with contextlib.closing(socket.socket()) as s: s.settimeout(GLOBAL_TIMEOUT) @@ -1528,8 +1649,9 @@ def test_idle_data_timeout2(self): data = self.client.sock.recv(BUFSIZE) self.assertEqual(data, b"421 Control connection timed out.\r\n") # ensure client has been kicked off - self.assertRaises((socket.error, EOFError), self.client.sendcmd, - 'noop') + self.assertRaises( + (socket.error, EOFError), self.client.sendcmd, 'noop' + ) def test_pasv_timeout(self): # Test pasv data channel timeout. The client which does not @@ -1573,6 +1695,7 @@ def test_disabled_port_timeout(self): class TestConfigurableOptions(PyftpdlibTestCase): """Test those daemon options which are commonly modified by user.""" + server_class = MProcessTestFTPd client_class = ftplib.FTP @@ -1619,25 +1742,40 @@ def test_max_connections(self): try: c1.connect(self.server.host, self.server.port) c2.connect(self.server.host, self.server.port) - self.assertRaises(ftplib.error_temp, c3.connect, self.server.host, - self.server.port) + self.assertRaises( + ftplib.error_temp, + c3.connect, + self.server.host, + self.server.port, + ) # with passive data channel established c2.quit() c1.login(USER, PASSWD) c1.makepasv() - self.assertRaises(ftplib.error_temp, c2.connect, self.server.host, - self.server.port) + self.assertRaises( + ftplib.error_temp, + c2.connect, + self.server.host, + self.server.port, + ) # with passive data socket waiting for connection c1.login(USER, PASSWD) c1.sendcmd('pasv') - self.assertRaises(ftplib.error_temp, c2.connect, self.server.host, - self.server.port) + self.assertRaises( + ftplib.error_temp, + c2.connect, + self.server.host, + self.server.port, + ) # with active data channel established c1.login(USER, PASSWD) with contextlib.closing(c1.makeport()): self.assertRaises( - ftplib.error_temp, c2.connect, self.server.host, - self.server.port) + ftplib.error_temp, + c2.connect, + self.server.host, + self.server.port, + ) finally: for c in (c1, c2, c3): try: @@ -1660,8 +1798,12 @@ def test_max_connections_per_ip(self): c1.connect(self.server.host, self.server.port) c2.connect(self.server.host, self.server.port) c3.connect(self.server.host, self.server.port) - self.assertRaises(ftplib.error_temp, c4.connect, self.server.host, - self.server.port) + self.assertRaises( + ftplib.error_temp, + c4.connect, + self.server.host, + self.server.port, + ) # Make sure client has been disconnected. # socket.error (Windows) or EOFError (Linux) exception is # supposed to be raised in such a case. @@ -1689,13 +1831,15 @@ def test_max_login_attempts(self): self.server.handler.auth_failed_timeout = 0 self.server.start() self.connect() - self.assertRaises(ftplib.error_perm, self.client.login, 'wrong', - 'wrong') + self.assertRaises( + ftplib.error_perm, self.client.login, 'wrong', 'wrong' + ) # socket.error (Windows) or EOFError (Linux) exceptions are # supposed to be raised when attempting to send/recv some data # using a disconnected socket - self.assertRaises((socket.error, EOFError), self.client.sendcmd, - 'noop') + self.assertRaises( + (socket.error, EOFError), self.client.sendcmd, 'noop' + ) def test_masquerade_address(self): # Test FTPHandler.masquerade_address attribute @@ -1709,8 +1853,9 @@ def test_masquerade_address(self): def test_masquerade_address_map(self): # Test FTPHandler.masquerade_address_map attribute self.server = self.server_class() - self.server.handler.masquerade_address_map = {self.server.host: - "128.128.128.128"} + self.server.handler.masquerade_address_map = { + self.server.host: "128.128.128.128" + } self.server.start() self.connect() host = ftplib.parse227(self.client.sendcmd('PASV'))[0] @@ -1817,11 +1962,13 @@ def on_file_received(self, file): def on_incomplete_file_sent(self, file): self.write( - "on_incomplete_file_sent:%s," % os.path.basename(file)) + "on_incomplete_file_sent:%s," % os.path.basename(file) + ) def on_incomplete_file_received(self, file): self.write( - "on_incomplete_file_received:%s," % os.path.basename(file)) + "on_incomplete_file_received:%s," % os.path.basename(file) + ) self.testfn = testfn = self.get_testfn() self.testfn2 = self.get_testfn() @@ -1855,14 +2002,13 @@ def test_on_logout_quit(self): self.client.login(USER, PASSWD) self.client.sendcmd('quit') self.read_file( - 'on_connect,on_login:%s,on_logout:%s,on_disconnect,' % ( - USER, USER)) + 'on_connect,on_login:%s,on_logout:%s,on_disconnect,' % (USER, USER) + ) def test_on_logout_rein(self): self.client.login(USER, PASSWD) self.client.sendcmd('rein') - self.read_file( - 'on_connect,on_login:%s,on_logout:%s,' % (USER, USER)) + self.read_file('on_connect,on_login:%s,on_logout:%s,' % (USER, USER)) def test_on_logout_no_pass(self): # make sure on_logout() is not called if USER was provided @@ -1876,12 +2022,14 @@ def test_on_logout_user_issued_twice(self): self.client.login(USER, PASSWD) self.client.login("anonymous") self.read_file( - 'on_connect,on_login:%s,on_logout:%s,on_login:anonymous,' % - (USER, USER)) + 'on_connect,on_login:%s,on_logout:%s,on_login:anonymous,' + % (USER, USER) + ) def test_on_login_failed(self): self.assertRaises( - ftplib.error_perm, self.client.login, 'foo', 'bar?!?') + ftplib.error_perm, self.client.login, 'foo', 'bar?!?' + ) self.read_file('on_connect,on_login_failed:foo+bar?!?,') def test_on_file_received(self): @@ -1892,8 +2040,9 @@ def test_on_file_received(self): self.client.login(USER, PASSWD) self.client.storbinary('stor ' + self.testfn2, dummyfile) self.read_file( - 'on_connect,on_login:%s,on_file_received:%s,' % ( - USER, self.testfn2)) + 'on_connect,on_login:%s,on_file_received:%s,' + % (USER, self.testfn2) + ) def test_on_file_sent(self): self.client.login(USER, PASSWD) @@ -1902,7 +2051,8 @@ def test_on_file_sent(self): f.write(data) self.client.retrbinary("retr " + self.testfn2, lambda x: x) self.read_file( - 'on_connect,on_login:%s,on_file_sent:%s,' % (USER, self.testfn2)) + 'on_connect,on_login:%s,on_file_sent:%s,' % (USER, self.testfn2) + ) @retry_on_failure() def test_on_incomplete_file_received(self): @@ -1912,7 +2062,8 @@ def test_on_incomplete_file_received(self): dummyfile.write(data) dummyfile.seek(0) with contextlib.closing( - self.client.transfercmd('stor ' + self.testfn2)) as conn: + self.client.transfercmd('stor ' + self.testfn2) + ) as conn: bytes_sent = 0 while True: chunk = dummyfile.read(BUFSIZE) @@ -1927,8 +2078,9 @@ def test_on_incomplete_file_received(self): self.assertRaises(ftplib.error_temp, self.client.getresp) # 426 self.assertEqual(self.client.getresp()[:3], "226") self.read_file( - 'on_connect,on_login:%s,on_incomplete_file_received:%s,' % - (USER, self.testfn2)) + 'on_connect,on_login:%s,on_incomplete_file_received:%s,' + % (USER, self.testfn2) + ) @retry_on_failure() def test_on_incomplete_file_sent(self): @@ -1937,8 +2089,9 @@ def test_on_incomplete_file_sent(self): with open(self.testfn2, 'wb') as f: f.write(data) bytes_recv = 0 - with contextlib.closing(self.client.transfercmd( - "retr " + self.testfn2, None)) as conn: + with contextlib.closing( + self.client.transfercmd("retr " + self.testfn2, None) + ) as conn: while True: chunk = conn.recv(BUFSIZE) bytes_recv += len(chunk) @@ -1946,8 +2099,9 @@ def test_on_incomplete_file_sent(self): break self.assertEqual(self.client.getline()[:3], "426") self.read_file( - 'on_connect,on_login:%s,on_incomplete_file_sent:%s,' % - (USER, self.testfn2)) + 'on_connect,on_login:%s,on_incomplete_file_sent:%s,' + % (USER, self.testfn2) + ) class _TestNetworkProtocols(object): # noqa @@ -1988,8 +2142,10 @@ def test_eprt(self): if not SUPPORTS_HYBRID_IPV6: # test wrong proto try: - self.client.sendcmd('eprt |%s|%s|%s|' % ( - self.other_proto, self.server.host, self.server.port)) + self.client.sendcmd( + 'eprt |%s|%s|%s|' + % (self.other_proto, self.server.host, self.server.port) + ) except ftplib.error_perm as err: self.assertEqual(str(err)[0:3], "522") else: @@ -2002,11 +2158,13 @@ def test_eprt(self): # len('|') < 3 self.assertEqual(self.cmdresp('eprt ||'), msg) # port > 65535 - self.assertEqual(self.cmdresp('eprt |%s|%s|65536|' % (self.proto, - self.HOST)), msg) + self.assertEqual( + self.cmdresp('eprt |%s|%s|65536|' % (self.proto, self.HOST)), msg + ) # port < 0 - self.assertEqual(self.cmdresp('eprt |%s|%s|-1|' % (self.proto, - self.HOST)), msg) + self.assertEqual( + self.cmdresp('eprt |%s|%s|-1|' % (self.proto, self.HOST)), msg + ) # port < 1024 resp = self.cmdresp('eprt |%s|%s|222|' % (self.proto, self.HOST)) self.assertEqual(resp[:3], '501') @@ -2051,10 +2209,12 @@ def test_epsv(self): # test connection for cmd in ('EPSV', 'EPSV ' + self.proto): - host, port = ftplib.parse229(self.client.sendcmd(cmd), - self.client.sock.getpeername()) + host, port = ftplib.parse229( + self.client.sendcmd(cmd), self.client.sock.getpeername() + ) with contextlib.closing( - socket.socket(self.client.af, socket.SOCK_STREAM)) as s: + socket.socket(self.client.af, socket.SOCK_STREAM) + ) as s: s.settimeout(GLOBAL_TIMEOUT) s.connect((host, port)) self.client.sendcmd('abor') @@ -2062,10 +2222,14 @@ def test_epsv(self): def test_epsv_all(self): self.client.sendcmd('epsv all') self.assertRaises(ftplib.error_perm, self.client.sendcmd, 'pasv') - self.assertRaises(ftplib.error_perm, self.client.sendport, self.HOST, - 2000) - self.assertRaises(ftplib.error_perm, self.client.sendcmd, - 'eprt |%s|%s|%s|' % (self.proto, self.HOST, 2000)) + self.assertRaises( + ftplib.error_perm, self.client.sendport, self.HOST, 2000 + ) + self.assertRaises( + ftplib.error_perm, + self.client.sendcmd, + 'eprt |%s|%s|%s|' % (self.proto, self.HOST, 2000), + ) @unittest.skipUnless(SUPPORTS_IPV4, "IPv4 not supported") @@ -2075,6 +2239,7 @@ class TestIPv4Environment(_TestNetworkProtocols, PyftpdlibTestCase): Runs tests contained in _TestNetworkProtocols class by using IPv4 plus some additional specific tests. """ + server_class = MProcessTestFTPd client_class = ftplib.FTP HOST = '127.0.0.1' @@ -2087,13 +2252,13 @@ def test_port_v4(self): # test bad arguments ae = self.assertEqual msg = "501 Invalid PORT format." - ae(self.cmdresp('port 127,0,0,1,1.1'), msg) # sep != ',' - ae(self.cmdresp('port X,0,0,1,1,1'), msg) # value != int + ae(self.cmdresp('port 127,0,0,1,1.1'), msg) # sep != ',' + ae(self.cmdresp('port X,0,0,1,1,1'), msg) # value != int ae(self.cmdresp('port 127,0,0,1,1,1,1'), msg) # len(args) > 6 - ae(self.cmdresp('port 127,0,0,1'), msg) # len(args) < 6 - ae(self.cmdresp('port 256,0,0,1,1,1'), msg) # oct > 255 + ae(self.cmdresp('port 127,0,0,1'), msg) # len(args) < 6 + ae(self.cmdresp('port 256,0,0,1,1,1'), msg) # oct > 255 ae(self.cmdresp('port 127,0,0,1,256,1'), msg) # port > 65535 - ae(self.cmdresp('port 127,0,0,1,-1,0'), msg) # port < 0 + ae(self.cmdresp('port 127,0,0,1,-1,0'), msg) # port < 0 # port < 1024 resp = self.cmdresp('port %s,1,1' % self.HOST.replace('.', ',')) self.assertEqual(resp[:3], '501') @@ -2111,7 +2276,8 @@ def test_eprt_v4(self): def test_pasv_v4(self): host, port = ftplib.parse227(self.client.sendcmd('pasv')) with contextlib.closing( - socket.socket(socket.AF_INET, socket.SOCK_STREAM)) as s: + socket.socket(socket.AF_INET, socket.SOCK_STREAM) + ) as s: s.settimeout(GLOBAL_TIMEOUT) s.connect((host, port)) @@ -2123,14 +2289,19 @@ class TestIPv6Environment(_TestNetworkProtocols, PyftpdlibTestCase): Runs tests contained in _TestNetworkProtocols class by using IPv6 plus some additional specific tests. """ + server_class = MProcessTestFTPd client_class = ftplib.FTP HOST = '::1' def test_port_v6(self): # PORT is not supposed to work - self.assertRaises(ftplib.error_perm, self.client.sendport, - self.server.host, self.server.port) + self.assertRaises( + ftplib.error_perm, + self.client.sendport, + self.server.host, + self.server.port, + ) def test_pasv_v6(self): # PASV is still supposed to work to support clients using @@ -2153,6 +2324,7 @@ class TestIPv6MixedEnvironment(PyftpdlibTestCase): What we are going to do here is starting the server in this manner and try to connect by using an IPv4 client. """ + server_class = MProcessTestFTPd client_class = ftplib.FTP HOST = "::" @@ -2198,7 +2370,8 @@ def test_eprt_v4(self): self.client.login(USER, PASSWD) # test connection with contextlib.closing( - socket.socket(socket.AF_INET, socket.SOCK_STREAM)) as sock: + socket.socket(socket.AF_INET, socket.SOCK_STREAM) + ) as sock: sock.bind((self.client.sock.getsockname()[0], 0)) sock.listen(5) sock.settimeout(2) @@ -2217,11 +2390,13 @@ def mlstline(cmd): self.client = self.client_class(timeout=GLOBAL_TIMEOUT) self.client.connect('127.0.0.1', self.server.port) self.client.login(USER, PASSWD) - host, port = ftplib.parse229(self.client.sendcmd('EPSV'), - self.client.sock.getpeername()) + host, port = ftplib.parse229( + self.client.sendcmd('EPSV'), self.client.sock.getpeername() + ) self.assertEqual('127.0.0.1', host) with contextlib.closing( - socket.socket(socket.AF_INET, socket.SOCK_STREAM)) as s: + socket.socket(socket.AF_INET, socket.SOCK_STREAM) + ) as s: s.settimeout(GLOBAL_TIMEOUT) s.connect((host, port)) self.assertTrue(mlstline('mlst /').endswith('/')) @@ -2231,6 +2406,7 @@ class TestCornerCases(PyftpdlibTestCase): """Tests for any kind of strange situation for the server to be in, mainly referring to bugs signaled on the bug tracker. """ + server_class = MProcessTestFTPd client_class = ftplib.FTP @@ -2282,8 +2458,11 @@ def connect(addr): # Set SO_LINGER to 1,0 causes a connection reset (RST) to # be sent when close() is called, instead of the standard # FIN shutdown sequence. - s.setsockopt(socket.SOL_SOCKET, socket.SO_LINGER, - struct.pack('ii', 1, 0)) + s.setsockopt( + socket.SOL_SOCKET, + socket.SO_LINGER, + struct.pack('ii', 1, 0), + ) s.settimeout(GLOBAL_TIMEOUT) try: s.connect(addr) @@ -2320,7 +2499,7 @@ def test_active_conn_error(self): with contextlib.closing(socket.socket()) as sock: sock.bind((HOST, 0)) port = sock.getsockname()[1] - self.client.sock.settimeout(.1) + self.client.sock.settimeout(0.1) try: resp = self.client.sendport(HOST, port) except ftplib.error_temp as err: @@ -2338,16 +2517,19 @@ def test_repr(self): str(inst) if hasattr(os, 'sendfile'): + def test_sendfile(self): # make sure that on python >= 3.3 we're using os.sendfile # rather than third party pysendfile module self.assertIs(sendfile, os.sendfile) if SUPPORTS_SENDFILE: + def test_sendfile_enabled(self): self.assertEqual(FTPHandler.use_sendfile, True) if hasattr(select, 'epoll') or hasattr(select, 'kqueue'): + def test_ioloop_fileno(self): fd = self.server.server.ioloop.fileno() self.assertIsInstance(fd, int, fd) @@ -2578,7 +2760,9 @@ def test_unforeseen_mdtm_event(self): self.assertRaisesRegex( ftplib.error_perm, "550 Can't determine file's last modification time", - self.client.sendcmd, 'mdtm ' + self.tempfile) + self.client.sendcmd, + 'mdtm ' + self.tempfile, + ) # make sure client hasn't been disconnected self.client.sendcmd('noop') finally: @@ -2592,8 +2776,9 @@ def test_stou_max_tries(self): class TestFS(AbstractedFS): def mkstemp(self, *args, **kwargs): - raise IOError(errno.EEXIST, - "No usable temporary file name found") + raise IOError( + errno.EEXIST, "No usable temporary file name found" + ) with self.server.lock: self.server.handler.abstracted_fs = TestFS @@ -2623,14 +2808,16 @@ def test_idle_timeout(self): data = self.client.sock.recv(BUFSIZE) self.assertEqual(data, b"421 Control connection timed out.\r\n") # ensure client has been kicked off - self.assertRaises((socket.error, EOFError), self.client.sendcmd, - 'noop') + self.assertRaises( + (socket.error, EOFError), self.client.sendcmd, 'noop' + ) finally: with self.server.lock: self.server.handler.timeout = 0.1 - @unittest.skipUnless(hasattr(socket, 'TCP_NODELAY'), - 'TCP_NODELAY not available') + @unittest.skipUnless( + hasattr(socket, 'TCP_NODELAY'), 'TCP_NODELAY not available' + ) @retry_on_failure() def test_tcp_no_delay(self): s = get_server_handler().socket @@ -2698,8 +2885,7 @@ def test_permit_privileged_ports(self): with self.server.lock: self.server.handler.permit_privileged_ports = False - self.assertRaises(ftplib.error_perm, self.client.sendport, HOST, - port) + self.assertRaises(ftplib.error_perm, self.client.sendport, HOST, port) if sock: port = sock.getsockname()[1] with self.server.lock: @@ -2711,8 +2897,7 @@ def test_permit_privileged_ports(self): s.close() @unittest.skipUnless(POSIX, "POSIX only") - @unittest.skipIf(not PY3 and sendfile is None, - "pysendfile not installed") + @unittest.skipIf(not PY3 and sendfile is None, "pysendfile not installed") @retry_on_failure() def test_sendfile_fails(self): # Makes sure that if sendfile() fails and no bytes were @@ -2722,10 +2907,12 @@ def test_sendfile_fails(self): self.dummy_sendfile.write(data) self.dummy_sendfile.seek(0) self.client.storbinary('stor ' + self.tempfile, self.dummy_sendfile) - with mock.patch('pyftpdlib.handlers.sendfile', - side_effect=OSError(errno.EINVAL)) as fun: + with mock.patch( + 'pyftpdlib.handlers.sendfile', side_effect=OSError(errno.EINVAL) + ) as fun: self.client.retrbinary( - 'retr ' + self.tempfile, self.dummy_recvfile.write) + 'retr ' + self.tempfile, self.dummy_recvfile.write + ) assert fun.called self.dummy_recvfile.seek(0) datafile = self.dummy_recvfile.read() @@ -2735,4 +2922,5 @@ def test_sendfile_fails(self): if __name__ == '__main__': from pyftpdlib.test.runner import run_from_name + run_from_name(__file__) diff --git a/pyftpdlib/test/test_functional_ssl.py b/pyftpdlib/test/test_functional_ssl.py index 7dc619a6..49fde4ff 100644 --- a/pyftpdlib/test/test_functional_ssl.py +++ b/pyftpdlib/test/test_functional_ssl.py @@ -37,8 +37,9 @@ from pyftpdlib.test.test_functional import TestTimeouts -CERTFILE = os.path.abspath(os.path.join(os.path.dirname(__file__), - 'keycert.pem')) +CERTFILE = os.path.abspath( + os.path.join(os.path.dirname(__file__), 'keycert.pem') +) del OpenSSL @@ -65,6 +66,7 @@ def login(self, *args, **kwargs): class FTPSServer(MProcessTestFTPd): """A threaded FTPS server used for functional testing.""" + handler = TLS_FTPHandler handler.certfile = CERTFILE @@ -167,11 +169,12 @@ class TestCornerCasesTLSMixin(TLSTestMixin, TestCornerCases): class TestFTPS(PyftpdlibTestCase): """Specific tests fot TSL_FTPHandler class.""" - def _setup(self, - tls_control_required=False, - tls_data_required=False, - ssl_protocol=ssl.PROTOCOL_SSLv23, - ): + def _setup( + self, + tls_control_required=False, + tls_data_required=False, + ssl_protocol=ssl.PROTOCOL_SSLv23, + ): self.server = FTPSServer() self.server.handler.tls_control_required = tls_control_required self.server.handler.tls_data_required = tls_data_required @@ -220,16 +223,18 @@ def test_auth(self): self.assertIsInstance(self.client.sock, ssl.SSLSocket) # AUTH issued twice msg = '503 Already using TLS.' - self.assertRaisesWithMsg(ftplib.error_perm, msg, - self.client.sendcmd, 'auth tls') + self.assertRaisesWithMsg( + ftplib.error_perm, msg, self.client.sendcmd, 'auth tls' + ) def test_pbsz(self): # unsecured self._setup() self.client.login(secure=False) msg = "503 PBSZ not allowed on insecure control connection." - self.assertRaisesWithMsg(ftplib.error_perm, msg, - self.client.sendcmd, 'pbsz 0') + self.assertRaisesWithMsg( + ftplib.error_perm, msg, self.client.sendcmd, 'pbsz 0' + ) # secured self.client.login(secure=True) resp = self.client.sendcmd('pbsz 0') @@ -239,8 +244,9 @@ def test_prot(self): self._setup() self.client.login(secure=False) msg = "503 PROT not allowed on insecure control connection." - self.assertRaisesWithMsg(ftplib.error_perm, msg, - self.client.sendcmd, 'prot p') + self.assertRaisesWithMsg( + ftplib.error_perm, msg, self.client.sendcmd, 'prot p' + ) self.client.login(secure=True) # secured self.client.prot_p() @@ -289,18 +295,21 @@ def test_unforseen_ssl_shutdown(self): def test_tls_control_required(self): self._setup(tls_control_required=True) msg = "550 SSL/TLS required on the control channel." - self.assertRaisesWithMsg(ftplib.error_perm, msg, - self.client.sendcmd, "user " + USER) - self.assertRaisesWithMsg(ftplib.error_perm, msg, - self.client.sendcmd, "pass " + PASSWD) + self.assertRaisesWithMsg( + ftplib.error_perm, msg, self.client.sendcmd, "user " + USER + ) + self.assertRaisesWithMsg( + ftplib.error_perm, msg, self.client.sendcmd, "pass " + PASSWD + ) self.client.login(secure=True) def test_tls_data_required(self): self._setup(tls_data_required=True) self.client.login(secure=True) msg = "550 SSL/TLS required on the data channel." - self.assertRaisesWithMsg(ftplib.error_perm, msg, - self.client.retrlines, 'list', lambda x: x) + self.assertRaisesWithMsg( + ftplib.error_perm, msg, self.client.retrlines, 'list', lambda x: x + ) self.client.prot_p() self.client.retrlines('list', lambda x: x) @@ -331,6 +340,7 @@ def try_protocol_combo(self, server_protocol, client_protocol): # self.try_protocol_combo(ssl.PROTOCOL_TLSv1, proto) if hasattr(ssl, "PROTOCOL_SSLv2"): + def test_sslv2(self): self.client.ssl_version = ssl.PROTOCOL_SSLv2 close_client(self.client) @@ -340,11 +350,13 @@ def test_sslv2(self): self.assertRaises(socket.error, self.client.login) else: with self.server.lock, self.assertRaises(socket.error): - self.client.connect(self.server.host, self.server.port, - timeout=0.1) + self.client.connect( + self.server.host, self.server.port, timeout=0.1 + ) self.client.ssl_version = ssl.PROTOCOL_SSLv2 if __name__ == '__main__': from pyftpdlib.test.runner import run_from_name + run_from_name(__file__) diff --git a/pyftpdlib/test/test_ioloop.py b/pyftpdlib/test/test_ioloop.py index 76d607ed..b9987c35 100644 --- a/pyftpdlib/test/test_ioloop.py +++ b/pyftpdlib/test/test_ioloop.py @@ -24,6 +24,7 @@ if hasattr(socket, 'socketpair'): socketpair = socket.socketpair else: + def socketpair(family=socket.AF_INET, type=socket.SOCK_STREAM, proto=0): with contextlib.closing(socket.socket(family, type, proto)) as ls: ls.bind(("localhost", 0)) @@ -138,8 +139,9 @@ def test_close_w_callback_exc(self): # Simulate an exception when close()ing the IO loop and a # scheduled callback raises an exception on cancel(). with mock.patch("pyftpdlib.ioloop.logger.error") as logerr: - with mock.patch("pyftpdlib.ioloop._CallLater.cancel", - side_effect=lambda: 1 / 0) as cancel: + with mock.patch( + "pyftpdlib.ioloop._CallLater.cancel", side_effect=lambda: 1 / 0 + ) as cancel: s = self.ioloop_class() self.addCleanup(s.close) s.call_later(1, lambda: 0) @@ -157,19 +159,22 @@ class DefaultIOLoopTestCase(PyftpdlibTestCase, BaseIOLoopTestCase): # select() # =================================================================== + class SelectIOLoopTestCase(PyftpdlibTestCase, BaseIOLoopTestCase): ioloop_class = pyftpdlib.ioloop.Select def test_select_eintr(self): # EINTR is supposed to be ignored - with mock.patch('pyftpdlib.ioloop.select.select', - side_effect=select.error()) as m: + with mock.patch( + 'pyftpdlib.ioloop.select.select', side_effect=select.error() + ) as m: m.side_effect.errno = errno.EINTR s, rd, wr = self.test_register() s.poll(0) # ...but just that - with mock.patch('pyftpdlib.ioloop.select.select', - side_effect=select.error()) as m: + with mock.patch( + 'pyftpdlib.ioloop.select.select', side_effect=select.error() + ) as m: m.side_effect.errno = errno.EBADF s, rd, wr = self.test_register() self.assertRaises(select.error, s.poll, 0) @@ -179,8 +184,10 @@ def test_select_eintr(self): # poll() # =================================================================== -@unittest.skipUnless(hasattr(pyftpdlib.ioloop, 'Poll'), - "poll() not available on this platform") + +@unittest.skipUnless( + hasattr(pyftpdlib.ioloop, 'Poll'), "poll() not available on this platform" +) class PollIOLoopTestCase(PyftpdlibTestCase, BaseIOLoopTestCase): ioloop_class = getattr(pyftpdlib.ioloop, "Poll", None) poller_mock = "pyftpdlib.ioloop.Poll._poller" @@ -210,35 +217,38 @@ def test_eintr_on_poll(self): def test_eexist_on_register(self): # EEXIST is supposed to be ignored with mock.patch(self.poller_mock, return_vaue=mock.Mock()) as m: - m.return_value.register.side_effect = \ - EnvironmentError(errno.EEXIST, "") + m.return_value.register.side_effect = EnvironmentError( + errno.EEXIST, "" + ) s, rd, wr = self.test_register() # ...but just that with mock.patch(self.poller_mock, return_vaue=mock.Mock()) as m: - m.return_value.register.side_effect = \ - EnvironmentError(errno.EBADF, "") + m.return_value.register.side_effect = EnvironmentError( + errno.EBADF, "" + ) self.assertRaises(EnvironmentError, self.test_register) def test_enoent_ebadf_on_unregister(self): # ENOENT and EBADF are supposed to be ignored for errnum in (errno.EBADF, errno.ENOENT): with mock.patch(self.poller_mock, return_vaue=mock.Mock()) as m: - m.return_value.unregister.side_effect = \ - EnvironmentError(errnum, "") + m.return_value.unregister.side_effect = EnvironmentError( + errnum, "" + ) s, rd, wr = self.test_register() s.unregister(rd) # ...but just those with mock.patch(self.poller_mock, return_vaue=mock.Mock()) as m: - m.return_value.unregister.side_effect = \ - EnvironmentError(errno.EEXIST, "") + m.return_value.unregister.side_effect = EnvironmentError( + errno.EEXIST, "" + ) s, rd, wr = self.test_register() self.assertRaises(EnvironmentError, s.unregister, rd) def test_enoent_on_modify(self): # ENOENT is supposed to be ignored with mock.patch(self.poller_mock, return_vaue=mock.Mock()) as m: - m.return_value.modify.side_effect = \ - OSError(errno.ENOENT, "") + m.return_value.modify.side_effect = OSError(errno.ENOENT, "") s, rd, wr = self.test_register() s.modify(rd, s.READ) @@ -247,8 +257,11 @@ def test_enoent_on_modify(self): # epoll() # =================================================================== -@unittest.skipUnless(hasattr(pyftpdlib.ioloop, 'Epoll'), - "epoll() not available on this platform (Linux only)") + +@unittest.skipUnless( + hasattr(pyftpdlib.ioloop, 'Epoll'), + "epoll() not available on this platform (Linux only)", +) class EpollIOLoopTestCase(PollIOLoopTestCase): ioloop_class = getattr(pyftpdlib.ioloop, "Epoll", None) poller_mock = "pyftpdlib.ioloop.Epoll._poller" @@ -258,8 +271,11 @@ class EpollIOLoopTestCase(PollIOLoopTestCase): # /dev/poll # =================================================================== -@unittest.skipUnless(hasattr(pyftpdlib.ioloop, 'DevPoll'), - "/dev/poll not available on this platform (Solaris only)") + +@unittest.skipUnless( + hasattr(pyftpdlib.ioloop, 'DevPoll'), + "/dev/poll not available on this platform (Solaris only)", +) class DevPollIOLoopTestCase(PyftpdlibTestCase, BaseIOLoopTestCase): ioloop_class = getattr(pyftpdlib.ioloop, "DevPoll", None) @@ -268,8 +284,11 @@ class DevPollIOLoopTestCase(PyftpdlibTestCase, BaseIOLoopTestCase): # kqueue # =================================================================== -@unittest.skipUnless(hasattr(pyftpdlib.ioloop, 'Kqueue'), - "/dev/poll not available on this platform (BSD only)") + +@unittest.skipUnless( + hasattr(pyftpdlib.ioloop, 'Kqueue'), + "/dev/poll not available on this platform (BSD only)", +) class KqueueIOLoopTestCase(PyftpdlibTestCase, BaseIOLoopTestCase): ioloop_class = getattr(pyftpdlib.ioloop, "Kqueue", None) @@ -317,6 +336,7 @@ def fun(x): # The test is reliable only on those systems where time.time() # provides time with a better precision than 1 second. if not str(time.time()).endswith('.0'): + def test_reset(self): def fun(x): ls.append(x) @@ -348,7 +368,8 @@ def fun(x): def test_errback(self): ls = [] self.ioloop.call_later( - 0.0, lambda: 1 // 0, _errback=lambda: ls.append(True)) + 0.0, lambda: 1 // 0, _errback=lambda: ls.append(True) + ) self.scheduler() self.assertEqual(ls, [True]) @@ -419,6 +440,7 @@ def fun(): # run it on systems where time.time() has a higher precision if POSIX: + def test_low_and_high_timeouts(self): # make sure a callback with a lower timeout is called more # frequently than another with a greater timeout @@ -454,7 +476,8 @@ def fun(): def test_errback(self): ls = [] self.ioloop.call_every( - 0.0, lambda: 1 // 0, _errback=lambda: ls.append(True)) + 0.0, lambda: 1 // 0, _errback=lambda: ls.append(True) + ) self.scheduler() self.assertTrue(ls) @@ -471,16 +494,20 @@ def get_connected_handler(self): def test_send_retry(self): ac = self.get_connected_handler() for errnum in pyftpdlib.ioloop._ERRNOS_RETRY: - with mock.patch("pyftpdlib.ioloop.socket.socket.send", - side_effect=socket.error(errnum, "")) as m: + with mock.patch( + "pyftpdlib.ioloop.socket.socket.send", + side_effect=socket.error(errnum, ""), + ) as m: self.assertEqual(ac.send(b"x"), 0) assert m.called def test_send_disconnect(self): ac = self.get_connected_handler() for errnum in pyftpdlib.ioloop._ERRNOS_DISCONNECTED: - with mock.patch("pyftpdlib.ioloop.socket.socket.send", - side_effect=socket.error(errnum, "")) as send: + with mock.patch( + "pyftpdlib.ioloop.socket.socket.send", + side_effect=socket.error(errnum, ""), + ) as send: with mock.patch.object(ac, "handle_close") as handle_close: self.assertEqual(ac.send(b"x"), 0) assert send.called @@ -489,16 +516,20 @@ def test_send_disconnect(self): def test_recv_retry(self): ac = self.get_connected_handler() for errnum in pyftpdlib.ioloop._ERRNOS_RETRY: - with mock.patch("pyftpdlib.ioloop.socket.socket.recv", - side_effect=socket.error(errnum, "")) as m: + with mock.patch( + "pyftpdlib.ioloop.socket.socket.recv", + side_effect=socket.error(errnum, ""), + ) as m: self.assertRaises(RetryError, ac.recv, 1024) assert m.called def test_recv_disconnect(self): ac = self.get_connected_handler() for errnum in pyftpdlib.ioloop._ERRNOS_DISCONNECTED: - with mock.patch("pyftpdlib.ioloop.socket.socket.recv", - side_effect=socket.error(errnum, "")) as send: + with mock.patch( + "pyftpdlib.ioloop.socket.socket.recv", + side_effect=socket.error(errnum, ""), + ) as send: with mock.patch.object(ac, "handle_close") as handle_close: self.assertEqual(ac.recv(b"x"), b'') assert send.called @@ -507,10 +538,11 @@ def test_recv_disconnect(self): def test_connect_af_unspecified_err(self): ac = AsyncChat() with mock.patch.object( - ac, "connect", - side_effect=socket.error(errno.EBADF, "")) as m: - self.assertRaises(socket.error, - ac.connect_af_unspecified, ("localhost", 0)) + ac, "connect", side_effect=socket.error(errno.EBADF, "") + ) as m: + self.assertRaises( + socket.error, ac.connect_af_unspecified, ("localhost", 0) + ) assert m.called self.assertIsNone(ac.socket) @@ -520,10 +552,11 @@ class TestAcceptor(PyftpdlibTestCase): def test_bind_af_unspecified_err(self): ac = Acceptor() with mock.patch.object( - ac, "bind", - side_effect=socket.error(errno.EBADF, "")) as m: - self.assertRaises(socket.error, - ac.bind_af_unspecified, ("localhost", 0)) + ac, "bind", side_effect=socket.error(errno.EBADF, "") + ) as m: + self.assertRaises( + socket.error, ac.bind_af_unspecified, ("localhost", 0) + ) assert m.called self.assertIsNone(ac.socket) @@ -531,8 +564,8 @@ def test_handle_accept_econnacorted(self): # https://github.com/giampaolo/pyftpdlib/issues/105 ac = Acceptor() with mock.patch.object( - ac, "accept", - side_effect=socket.error(errno.ECONNABORTED, "")) as m: + ac, "accept", side_effect=socket.error(errno.ECONNABORTED, "") + ) as m: ac.handle_accept() assert m.called self.assertIsNone(ac.socket) @@ -548,4 +581,5 @@ def test_handle_accept_typeerror(self): if __name__ == '__main__': from pyftpdlib.test.runner import run_from_name + run_from_name(__file__) diff --git a/pyftpdlib/test/test_misc.py b/pyftpdlib/test/test_misc.py index e4896136..5f0ba311 100644 --- a/pyftpdlib/test/test_misc.py +++ b/pyftpdlib/test/test_misc.py @@ -25,6 +25,7 @@ class TestCommandLineParser(PyftpdlibTestCase): """Test command line parser.""" + SYSARGV = sys.argv STDERR = sys.stderr @@ -41,6 +42,7 @@ def serve_forever(self, *args, **kwargs): if PY3: import io + self.devnull = io.StringIO() else: self.devnull = BytesIO() @@ -150,4 +152,5 @@ def test_D_option(self): if __name__ == '__main__': from pyftpdlib.test.runner import run_from_name + run_from_name(__file__) diff --git a/pyftpdlib/test/test_servers.py b/pyftpdlib/test/test_servers.py index e99ad6e4..3f6f13ed 100644 --- a/pyftpdlib/test/test_servers.py +++ b/pyftpdlib/test/test_servers.py @@ -39,6 +39,7 @@ class TestFTPServer(PyftpdlibTestCase): """Tests for *FTPServer classes.""" + server_class = ThreadedTestFTPd client_class = ftplib.FTP @@ -92,8 +93,9 @@ class ThreadFTPTestMixin: server_class = _TFTPd -class TestFtpAuthenticationThreadMixin(ThreadFTPTestMixin, - TestFtpAuthentication): +class TestFtpAuthenticationThreadMixin( + ThreadFTPTestMixin, TestFtpAuthentication +): pass @@ -156,19 +158,23 @@ class TestCornerCasesThreadMixin(ThreadFTPTestMixin, TestCornerCases): # ===================================================================== if MPROCESS_SUPPORT: + class MultiProcFTPd(ThreadedTestFTPd): server_class = servers.MultiprocessFTPServer class MProcFTPTestMixin: server_class = MultiProcFTPd + else: + @unittest.skipIf(True, "multiprocessing module not installed") class MProcFTPTestMixin: pass -class TestFtpAuthenticationMProcMixin(MProcFTPTestMixin, - TestFtpAuthentication): +class TestFtpAuthenticationMProcMixin( + MProcFTPTestMixin, TestFtpAuthentication +): pass diff --git a/pyproject.toml b/pyproject.toml index 15fe7527..513d7107 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,3 +1,11 @@ +[tool.black] +target-version = ["py37"] +line-length = 79 +skip-string-normalization = true +# https://black.readthedocs.io/en/stable/the_black_code_style/future_style.html +preview = true +enable-unstable-feature = ["multiline_string_handling", "string_processing", "wrap_long_dict_values_in_parens"] + [tool.ruff] # https://beta.ruff.rs/docs/settings/ target-version = "py37" @@ -22,6 +30,7 @@ ignore = [ "COM812", # Trailing comma missing "D", # pydocstyle "DTZ", # flake8-datetimez + "E203", # [*] Whitespace before ':' (clashes with black) "EM", # flake8-errmsg "ERA001", # Found commented-out code "F841", # Local variable `wr` is assigned to but never used diff --git a/scripts/internal/generate_manifest.py b/scripts/internal/generate_manifest.py index 0f96dd1b..88249a37 100755 --- a/scripts/internal/generate_manifest.py +++ b/scripts/internal/generate_manifest.py @@ -13,7 +13,7 @@ SKIP_EXTS = ('.png', '.jpg', '.jpeg', '.svg') -SKIP_FILES = ('appveyor.yml') +SKIP_FILES = 'appveyor.yml' SKIP_PREFIXES = ('.ci/', '.github/') @@ -24,9 +24,11 @@ def sh(cmd): def main(): files = sh(["git", "ls-files"]).split('\n') for file in files: - if file.startswith(SKIP_PREFIXES) or \ - os.path.splitext(file)[1].lower() in SKIP_EXTS or \ - file in SKIP_FILES: + if ( + file.startswith(SKIP_PREFIXES) + or os.path.splitext(file)[1].lower() in SKIP_EXTS + or file in SKIP_FILES + ): continue print("include " + file) diff --git a/scripts/internal/print_announce.py b/scripts/internal/print_announce.py index e93e68ea..fe4c92ce 100755 --- a/scripts/internal/print_announce.py +++ b/scripts/internal/print_announce.py @@ -21,8 +21,9 @@ PRJ_URL_HOME = 'https://github.com/giampaolo/pyftpdlib' PRJ_URL_DOC = 'http://pyftpdlib.readthedocs.io' PRJ_URL_DOWNLOAD = 'https://pypi.python.org/pypi/pyftpdlib' -PRJ_URL_WHATSNEW = \ +PRJ_URL_WHATSNEW = ( 'https://github.com/giampaolo/pyftpdlib/blob/master/HISTORY.rst' +) template = """\ Hello all, @@ -92,15 +93,17 @@ def get_changes(): def main(): changes = get_changes() - print(template.format( - prj_name=PRJ_NAME, - prj_version=PRJ_VERSION, - prj_urlhome=PRJ_URL_HOME, - prj_urldownload=PRJ_URL_DOWNLOAD, - prj_urldoc=PRJ_URL_DOC, - prj_urlwhatsnew=PRJ_URL_WHATSNEW, - changes=changes, - )) + print( + template.format( + prj_name=PRJ_NAME, + prj_version=PRJ_VERSION, + prj_urlhome=PRJ_URL_HOME, + prj_urldownload=PRJ_URL_DOWNLOAD, + prj_urldoc=PRJ_URL_DOC, + prj_urlwhatsnew=PRJ_URL_WHATSNEW, + changes=changes, + ) + ) if __name__ == '__main__': diff --git a/scripts/internal/winmake.py b/scripts/internal/winmake.py index 844f6236..fdf83d62 100755 --- a/scripts/internal/winmake.py +++ b/scripts/internal/winmake.py @@ -87,7 +87,7 @@ def safe_print(text, file=sys.stdout): def stderr_handle(): GetStdHandle = ctypes.windll.Kernel32.GetStdHandle - STD_ERROR_HANDLE_ID = ctypes.c_ulong(0xfffffff4) + STD_ERROR_HANDLE_ID = ctypes.c_ulong(0xFFFFFFF4) GetStdHandle.restype = ctypes.c_ulong handle = GetStdHandle(STD_ERROR_HANDLE_ID) atexit.register(ctypes.windll.Kernel32.CloseHandle, handle) @@ -116,6 +116,7 @@ def sh(cmd, nolog=False): def rm(pattern, directory=False): """Recursively remove a file or dir by pattern.""" + def safe_remove(path): try: os.remove(path) @@ -432,9 +433,11 @@ def install_git_hooks(): """Install GIT pre-commit hook.""" if os.path.isdir('.git'): src = os.path.join( - ROOT_DIR, "scripts", "internal", "git_pre_commit.py") + ROOT_DIR, "scripts", "internal", "git_pre_commit.py" + ) dst = os.path.realpath( - os.path.join(ROOT_DIR, ".git", "hooks", "pre-commit")) + os.path.join(ROOT_DIR, ".git", "hooks", "pre-commit") + ) with open(src) as s, open(dst, "w") as d: d.write(s.read()) @@ -472,9 +475,7 @@ def main(): global PYTHON parser = argparse.ArgumentParser() # option shared by all commands - parser.add_argument( - '-p', '--python', - help="use python executable path") + parser.add_argument('-p', '--python', help="use python executable path") sp = parser.add_subparsers(dest='command', title='targets') sp.add_parser('build', help="build") sp.add_parser('clean', help="deletes dev files") @@ -505,7 +506,8 @@ def main(): PYTHON = get_python(args.python) if not PYTHON: return sys.exit( - "can't find any python installation matching %r" % args.python) + "can't find any python installation matching %r" % args.python + ) os.putenv('PYTHON', PYTHON) win_colorprint("using " + PYTHON) diff --git a/setup.py b/setup.py index a7fdaf55..dee1f04b 100644 --- a/setup.py +++ b/setup.py @@ -22,8 +22,9 @@ def get_version(): - INIT = os.path.abspath(os.path.join(os.path.dirname(__file__), - 'pyftpdlib', '__init__.py')) + INIT = os.path.abspath( + os.path.join(os.path.dirname(__file__), 'pyftpdlib', '__init__.py') + ) with open(INIT) as f: for line in f: if line.startswith('__ver__'): @@ -38,6 +39,7 @@ def get_version(): def term_supports_colors(): try: import curses + assert sys.stderr.isatty() curses.setupterm() assert curses.tigetnum("colors") > 0 @@ -67,7 +69,7 @@ def hilite(s, ok=True, bold=False): if sys.version_info < (2, 7): # noqa sys.exit('python version not supported (< 2.7)') -require_pysendfile = (os.name == 'posix' and sys.version_info < (3, 3)) +require_pysendfile = os.name == 'posix' and sys.version_info < (3, 3) extras_require = {'ssl': ["PyOpenSSL"]} if require_pysendfile: @@ -98,10 +100,12 @@ def main(): 'keycert.pem', ], }, + # fmt: off keywords=['ftp', 'ftps', 'server', 'ftpd', 'daemon', 'python', 'ssl', 'sendfile', 'asynchronous', 'nonblocking', 'eventdriven', 'rfc959', 'rfc1123', 'rfc2228', 'rfc2428', 'rfc2640', 'rfc3659'], + # fmt: on extras_require=extras_require, classifiers=[ 'Development Status :: 5 - Production/Stable', @@ -129,6 +133,7 @@ def main(): # fallback on using third-party pysendfile module # https://github.com/giampaolo/pysendfile/ import sendfile + if hasattr(sendfile, 'has_sf_hdtr'): # old 1.2.4 version raise ImportError except ImportError: