diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 1db81a50..c5633473 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -33,7 +33,7 @@ jobs: - name: Install pydeps ${{ matrix.os }} if: matrix.os != 'windows-latest' run: | - make setup-dev-env + make install-pydeps-test - name: Tests run: | diff --git a/MANIFEST.in b/MANIFEST.in index 31e8cff6..0fcc04b4 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -59,6 +59,7 @@ include scripts/ftpbench include scripts/internal/check_broken_links.py include scripts/internal/generate_manifest.py include scripts/internal/git_pre_commit.py +include scripts/internal/install_pip.py include scripts/internal/print_announce.py include scripts/internal/purge_installation.py include scripts/internal/winmake.py diff --git a/Makefile b/Makefile index bbba11b9..8885c234 100644 --- a/Makefile +++ b/Makefile @@ -4,40 +4,24 @@ # To run a specific test: # $ make test ARGS="-v -s pyftpdlib/test/test_functional.py::TestIPv6MixedEnvironment::test_port_v4" +# Configurable PYTHON = python3 ARGS = -# mandatory deps for running tests -PYDEPS = \ - psutil \ - pyopenssl \ - pytest \ - pytest-xdist \ - setuptools -# dev deps -ifndef GITHUB_ACTIONS - PYDEPS += \ - black \ - check-manifest \ - coverage \ - pylint \ - pytest-cov \ - pytest-xdist \ - rstcheck \ - ruff \ - toml-sort \ - twine -endif - # In not in a virtualenv, add --user options for install commands. -INSTALL_OPTS = `$(PYTHON) -c "import sys; print('' if hasattr(sys, 'real_prefix') else '--user')"` +SETUP_INSTALL_ARGS = `$(PYTHON) -c \ + "import sys; print('' if hasattr(sys, 'real_prefix') or hasattr(sys, 'base_prefix') and sys.base_prefix != sys.prefix else '--user')"` TEST_PREFIX = PYTHONWARNINGS=always PYTEST_ARGS = -v -s --tb=short NUM_WORKERS = `$(PYTHON) -c "import os; print(os.cpu_count() or 1)"` +PIP_INSTALL_ARGS = --trusted-host files.pythonhosted.org --trusted-host pypi.org --upgrade # if make is invoked with no arg, default to `make help` .DEFAULT_GOAL := help +# install git hook +_ := $(shell mkdir -p .git/hooks/ && ln -sf ../../scripts/internal/git_pre_commit.py .git/hooks/pre-commit && chmod +x .git/hooks/pre-commit) + # =================================================================== # Install # =================================================================== @@ -74,37 +58,25 @@ clean: ## Remove all build files. install: ## Install this package. # make sure setuptools is installed (needed for 'develop' / edit mode) $(PYTHON) -c "import setuptools" - $(PYTHON) setup.py develop $(INSTALL_OPTS) + $(PYTHON) setup.py develop $(SETUP_INSTALL_ARGS) uninstall: ## Uninstall this package. cd ..; $(PYTHON) -m pip uninstall -y -v pyftpdlib || true $(PYTHON) scripts/internal/purge_installation.py install-pip: ## Install pip (no-op if already installed). - @$(PYTHON) -c \ - "import sys, ssl, os, pkgutil, tempfile, atexit; \ - from urllib.request import urlopen; \ - sys.exit(0) if pkgutil.find_loader('pip') else None; \ - exec(pyexc); \ - ctx = ssl._create_unverified_context() if hasattr(ssl, '_create_unverified_context') else None; \ - url = 'https://bootstrap.pypa.io/get-pip.py'; \ - kw = dict(context=ctx) if ctx else {}; \ - req = urlopen(url, **kw); \ - data = req.read(); \ - f = tempfile.NamedTemporaryFile(suffix='.py'); \ - atexit.register(f.close); \ - f.write(data); \ - f.flush(); \ - print('downloaded %s' % f.name); \ - code = os.system('%s %s --user --upgrade' % (sys.executable, f.name)); \ - f.close(); \ - sys.exit(code);" - -setup-dev-env: ## Install GIT hooks, pip, test deps (also upgrades them). + $(PYTHON) scripts/internal/install_pip.py + +install-pydeps-test: ## Install python deps necessary to run unit tests. + ${MAKE} install-pip + $(PYTHON) -m pip install $(PIP_INSTALL_ARGS) pip # upgrade pip to latest version + $(PYTHON) -m pip install $(PIP_INSTALL_ARGS) `$(PYTHON) -c "import setup; print(' '.join(setup.TEST_DEPS))"` + +install-pydeps-dev: ## Install python deps meant for local development. ${MAKE} install-git-hooks ${MAKE} install-pip - $(PYTHON) -m pip install $(INSTALL_OPTS) --upgrade pip setuptools - $(PYTHON) -m pip install $(INSTALL_OPTS) --upgrade $(PYDEPS) + $(PYTHON) -m pip install $(PIP_INSTALL_ARGS) pip # upgrade pip to latest version + $(PYTHON) -m pip install $(PIP_INSTALL_ARGS) `$(PYTHON) -c "import setup; print(' '.join(setup.TEST_DEPS + setup.DEV_DEPS))"` # =================================================================== # Tests diff --git a/scripts/internal/install_pip.py b/scripts/internal/install_pip.py new file mode 100755 index 00000000..eb89b117 --- /dev/null +++ b/scripts/internal/install_pip.py @@ -0,0 +1,58 @@ +#!/usr/bin/env python3 + +# Copyright (C) 2007 Giampaolo Rodola' . +# Use of this source code is governed by MIT license that can be +# found in the LICENSE file. + +import sys + + +try: + import pip # noqa: F401 +except ImportError: + pass +else: + print("pip already installed") + sys.exit(0) + +import os +import ssl +import tempfile + + +PY3 = sys.version_info[0] >= 3 +if PY3: + from urllib.request import urlopen + + URL = "https://bootstrap.pypa.io/get-pip.py" + +else: + from urllib2 import urlopen + + URL = "https://bootstrap.pypa.io/pip/2.7/get-pip.py" + + +def main(): + ssl_context = ( + ssl._create_unverified_context() + if hasattr(ssl, "_create_unverified_context") + else None + ) + with tempfile.NamedTemporaryFile(suffix=".py") as f: + print("downloading %s into %s" % (URL, f.name)) + kwargs = dict(context=ssl_context) if ssl_context else {} + req = urlopen(URL, **kwargs) + data = req.read() + req.close() + + f.write(data) + f.flush() + print("download finished, installing pip") + + code = os.system("%s %s --user --upgrade" % (sys.executable, f.name)) + + sys.exit(code) + + +if __name__ == "__main__": + main() diff --git a/scripts/internal/winmake.py b/scripts/internal/winmake.py index 56396411..c111b6b6 100755 --- a/scripts/internal/winmake.py +++ b/scripts/internal/winmake.py @@ -20,28 +20,25 @@ import os import shutil import site -import ssl import subprocess import sys -import tempfile -from urllib.request import urlopen PYTHON = os.getenv('PYTHON', sys.executable) -GET_PIP_URL = "https://bootstrap.pypa.io/get-pip.py" -PYTEST_ARGS = "-v --tb=native -o" +PY3 = sys.version_info[0] >= 3 +PYTEST_ARGS = "-v -s --tb=short" HERE = os.path.abspath(os.path.dirname(__file__)) ROOT_DIR = os.path.realpath(os.path.join(HERE, "..", "..")) PYPY = '__pypy__' in sys.builtin_module_names -DEPS = [ - "pip", - "psutil", - "pytest", - "pyopenssl", - "pypiwin32", - "setuptools", - "wmi", -] +WINDOWS = os.name == "nt" + + +sys.path.insert(0, ROOT_DIR) # so that we can import setup.py + +import setup # NOQA + +TEST_DEPS = setup.TEST_DEPS +DEV_DEPS = setup.DEV_DEPS _cmds = {} @@ -86,6 +83,8 @@ def stderr_handle(): def win_colorprint(s, color=LIGHTBLUE): + if not WINDOWS: + return print(s) color += 8 # bold handle = stderr_handle() SetConsoleTextAttribute = ctypes.windll.Kernel32.SetConsoleTextAttribute @@ -99,7 +98,9 @@ def win_colorprint(s, color=LIGHTBLUE): def sh(cmd, nolog=False): if not nolog: safe_print("cmd: " + cmd) - p = subprocess.Popen(cmd, shell=True, env=os.environ, cwd=os.getcwd()) + p = subprocess.Popen( # noqa S602 + cmd, shell=True, env=os.environ, cwd=os.getcwd() + ) p.communicate() if p.returncode != 0: sys.exit(p.returncode) @@ -108,6 +109,15 @@ def sh(cmd, nolog=False): def rm(pattern, directory=False): """Recursively remove a file or dir by pattern.""" + def safe_remove(path): + try: + os.remove(path) + except OSError as err: + if err.errno != errno.ENOENT: + raise + else: + safe_print("rm %s" % path) + def safe_rmtree(path): def onerror(fun, path, excinfo): exc = excinfo[1] @@ -126,11 +136,11 @@ def onerror(fun, path, excinfo): safe_remove(pattern) return - for root, subdirs, subfiles in os.walk('.'): + for root, dirs, files in os.walk('.'): root = os.path.normpath(root) if root.startswith('.git/'): continue - found = fnmatch.filter(subdirs if directory else subfiles, pattern) + found = fnmatch.filter(dirs if directory else files, pattern) for name in found: path = os.path.join(root, name) if directory: @@ -164,15 +174,15 @@ def onerror(fun, path, excinfo): def recursive_rm(*patterns): """Recursively remove a file or matching a list of patterns.""" - for root, subdirs, subfiles in os.walk('.'): + for root, dirs, files in os.walk('.'): root = os.path.normpath(root) if root.startswith('.git/'): continue - for file in subfiles: + for file in files: for pattern in patterns: if fnmatch.fnmatch(file, pattern): safe_remove(os.path.join(root, file)) - for dir in subdirs: + for dir in dirs: for pattern in patterns: if fnmatch.fnmatch(dir, pattern): safe_rmtree(os.path.join(root, dir)) @@ -231,26 +241,7 @@ def upload_wheels(): def install_pip(): """Install pip.""" - try: - sh(f'{PYTHON} -c "import pip"') - except SystemExit: - if hasattr(ssl, '_create_unverified_context'): - ctx = ssl._create_unverified_context() - else: - ctx = None - kw = dict(context=ctx) if ctx else {} - safe_print(f"downloading {GET_PIP_URL}") - req = urlopen(GET_PIP_URL, **kw) - data = req.read() - - tfile = os.path.join(tempfile.gettempdir(), 'get-pip.py') - with open(tfile, 'wb') as f: - f.write(data) - - try: - sh(f'{PYTHON} {tfile} --user') - finally: - os.remove(tfile) + sh('%s %s' % (PYTHON, os.path.join(HERE, "install_pip.py"))) def install(): @@ -326,20 +317,18 @@ def clean(): safe_rmtree("tmp") -def setup_dev_env(): +def install_pydeps_test(): """Install useful deps.""" install_pip() install_git_hooks() - sh(f"{PYTHON} -m pip install -U {' '.join(DEPS)}") + sh("%s -m pip install --user -U %s" % (PYTHON, " ".join(TEST_DEPS))) -def lint(): - """Run flake8 against all py files.""" - py_files = subprocess.check_output("git ls-files") - py_files = py_files.decode() - py_files = [x for x in py_files.split() if x.endswith('.py')] - py_files = ' '.join(py_files) - sh(f"{PYTHON} -m flake8 {py_files}", nolog=True) +def install_pydeps_dev(): + """Install useful deps.""" + install_pip() + install_git_hooks() + sh("%s -m pip install --user -U %s" % (PYTHON, " ".join(DEV_DEPS))) def test(args=""): @@ -398,7 +387,7 @@ def test_by_name(name): test(name) -def test_failed(): +def test_last_failed(): """Re-run tests which failed on last run.""" build() sh(f"{PYTHON} -m pytest {PYTEST_ARGS} --last-failed") @@ -446,8 +435,7 @@ def get_python(path): return pypath -def main(): - global PYTHON +def parse_args(): parser = argparse.ArgumentParser() # option shared by all commands parser.add_argument('-p', '--python', help="use python executable path") @@ -459,6 +447,8 @@ def main(): sp.add_parser('install', help="build + install in develop/edit mode") sp.add_parser('install-git-hooks', help="install GIT pre-commit hook") sp.add_parser('install-pip', help="install pip") + sp.add_parser('install-pydeps-dev', help="install dev python deps") + sp.add_parser('install-pydeps-test', help="install python test deps") sp.add_parser('test', help="run tests") sp.add_parser('test-authorizers') sp.add_parser('test-filesystems') @@ -468,28 +458,34 @@ def main(): sp.add_parser('test-misc') sp.add_parser('test-servers') sp.add_parser('lint', help="run flake8 against all py files") - sp.add_parser('setup-dev-env', help="install deps") test = sp.add_parser('test', help="[ARG] run tests") test_by_name = sp.add_parser('test-by-name', help=" run test by name") sp.add_parser('uninstall', help="uninstall") for p in (test, test_by_name): p.add_argument('arg', type=str, nargs='?', default="", help="arg") + args = parser.parse_args() + if not args.command or args.command == 'help': + parser.print_help(sys.stderr) + sys.exit(1) + + return args + + +def main(): + global PYTHON + args = parse_args() # set python exe PYTHON = get_python(args.python) if not PYTHON: return sys.exit( - f"can't find any python installation matching {args.python!r}" + "can't find any python installation matching %r" % args.python ) os.putenv('PYTHON', PYTHON) win_colorprint("using " + PYTHON) - if not args.command or args.command == 'help': - parser.print_help(sys.stderr) - sys.exit(1) - fname = args.command.replace('-', '_') fun = getattr(sys.modules[__name__], fname) # err if fun not defined funargs = [] diff --git a/setup.py b/setup.py index 0b9da326..3c234ecd 100644 --- a/setup.py +++ b/setup.py @@ -7,7 +7,6 @@ $ python setup.py install """ - import ast import os import sys @@ -15,11 +14,43 @@ try: + import setuptools from setuptools import setup except ImportError: + setuptools = None from distutils.core import setup +WINDOWS = os.name == "nt" + +# Test deps, installable via `pip install .[test]`. +TEST_DEPS = [ + "psutil", + "pyopenssl", + "pytest", + "pytest-xdist", + "setuptools", +] +if WINDOWS: + TEST_DEPS.append("pywin32") + +# Development deps, installable via `pip install .[dev]`. +DEV_DEPS = [ + "black", + "check-manifest", + "coverage", + "pylint", + "pytest-cov", + "pytest-xdist", + "rstcheck", + "ruff", + "toml-sort", + "twine", +] +if WINDOWS: + DEV_DEPS.extend(["pyreadline", "pdbpp"]) + + def get_version(): INIT = os.path.abspath( os.path.join(os.path.dirname(__file__), 'pyftpdlib', '__init__.py') @@ -65,22 +96,12 @@ def hilite(s, ok=True, bold=False): return f"\x1b[{';'.join(attr)}m{s}\x1b[0m" -if sys.version_info[0] < 3: # noqa: UP036 - sys.exit( - 'Python 2 is no longer supported. Latest version is 1.5.10; use:\n' - 'python3 -m pip install pyftpdlib==1.5.10' - ) - -extras_require = {'ssl': ["PyOpenSSL"]} - -VERSION = get_version() - with open('README.rst') as f: long_description = f.read() def main(): - setup( + kwargs = dict( name='pyftpdlib', version=get_version(), description='Very fast asynchronous FTP server library', @@ -108,7 +129,6 @@ def main(): "pyasyncore;python_version>='3.12'", "pyasynchat;python_version>='3.12'", ], - extras_require=extras_require, classifiers=[ 'Development Status :: 5 - Production/Stable', 'Environment :: Console', @@ -125,6 +145,22 @@ def main(): ], ) + if setuptools is not None: + extras_require = { + "dev": DEV_DEPS, + "test": TEST_DEPS, + "ssl": "PyOpenSSL", + } + kwargs.update( + python_requires=( + ">2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, !=3.5.*" + ), + extras_require=extras_require, + zip_safe=False, + ) + + setup(**kwargs) + try: from OpenSSL import SSL # NOQA except ImportError: @@ -135,5 +171,11 @@ def main(): print(hilite(msg, ok=False), file=sys.stderr) +if sys.version_info[0] < 3: # noqa: UP036 + sys.exit( + 'Python 2 is no longer supported. Latest version is 1.5.10; use:\n' + 'python3 -m pip install pyftpdlib==1.5.10' + ) + if __name__ == '__main__': main()