From f7778378fc05803b3d96dfa7b2f40c0aac9ca710 Mon Sep 17 00:00:00 2001 From: Josh Whitley Date: Thu, 6 Oct 2022 16:19:08 -0500 Subject: [PATCH 1/3] Handle pip packages as version specifiers. Signed-off-by: Joshua Whitley --- setup.py | 2 +- src/rosdep2/platforms/pip.py | 34 +++++++++++++++++++++++++++------- 2 files changed, 28 insertions(+), 8 deletions(-) diff --git a/setup.py b/setup.py index 76ee73bb1..1943811c1 100644 --- a/setup.py +++ b/setup.py @@ -50,6 +50,6 @@ kwargs['name'] += '_modules' kwargs['entry_points'] = {} else: - kwargs['install_requires'] += ['catkin_pkg >= 0.4.0', 'rospkg >= 1.4.0', 'rosdistro >= 0.7.5'] + kwargs['install_requires'] += ['catkin_pkg >= 0.4.0', 'rospkg >= 1.4.0', 'rosdistro >= 0.7.5', 'packaging'] setup(**kwargs) diff --git a/src/rosdep2/platforms/pip.py b/src/rosdep2/platforms/pip.py index 8dfbb6927..3832b2020 100644 --- a/src/rosdep2/platforms/pip.py +++ b/src/rosdep2/platforms/pip.py @@ -34,6 +34,8 @@ import subprocess import sys +from packaging.requirements import Requirement +from packaging.version import Version from ..core import InstallFailed from ..installers import PackageManagerInstaller from ..shell_utils import read_stdout @@ -79,7 +81,8 @@ def is_cmd_available(cmd): def pip_detect(pkgs, exec_fn=None): """ - Given a list of package, return the list of installed packages. + Given a list of package specifications, return the list of installed + packages which meet the specifications. :param exec_fn: function to execute Popen and read stdout (for testing) """ @@ -94,10 +97,20 @@ def pip_detect(pkgs, exec_fn=None): pkg_list = exec_fn(pip_cmd + ['freeze']).split('\n') ret_list = [] + version_list = [] + req_list = [] + for pkg in pkg_list: pkg_row = pkg.split('==') - if pkg_row[0] in pkgs: - ret_list.append(pkg_row[0]) + version_list.append((pkg_row[0], Version(pkg_row[1]))) + + for pkg in pkgs: + req_list.append(Requirement(pkg)) + + for req in req_list: + for pkg in [ver for ver in version_list if ver[0] == req.name]: + if pkg[1] in req.specifier: + ret_list.append(req.name) # Try to detect with the return code of `pip show`. # This can show the existance of things like `argparse` which @@ -105,18 +118,25 @@ def pip_detect(pkgs, exec_fn=None): # See: # https://github.com/pypa/pip/issues/1570#issuecomment-71111030 if fallback_to_pip_show: - for pkg in [p for p in pkgs if p not in ret_list]: + for req in [r for r in req_list if r.name not in ret_list]: # does not see retcode but stdout for old pip to check if installed proc = subprocess.Popen( - pip_cmd + ['show', pkg], + pip_cmd + ['show', req.name], stdout=subprocess.PIPE, stderr=subprocess.STDOUT ) output, _ = proc.communicate() output = output.strip() if proc.returncode == 0 and output: - # `pip show` detected it, add it to the list. - ret_list.append(pkg) + # `pip show` detected it, check the version. + show_split = output.split('\n') + + for line in [l for l in show_split if l.startswith('Version:')]: + version = Version(line.strip().split()[1]) + + if version in req.specifier: + # version matches, add it to the list + ret_list.append(req.name) return ret_list From 04f7da2a2f7457925fddd677ce35d64fc8b5d5d2 Mon Sep 17 00:00:00 2001 From: Joshua Whitley Date: Fri, 7 Oct 2022 13:06:37 -0500 Subject: [PATCH 2/3] Handle parsing of unusual instances in output of pip freeze. --- src/rosdep2/platforms/pip.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/rosdep2/platforms/pip.py b/src/rosdep2/platforms/pip.py index 3832b2020..89b0d3dcd 100644 --- a/src/rosdep2/platforms/pip.py +++ b/src/rosdep2/platforms/pip.py @@ -102,6 +102,10 @@ def pip_detect(pkgs, exec_fn=None): for pkg in pkg_list: pkg_row = pkg.split('==') + + # Account for some unusual instances of === instead of == + pkg_row[1] = pkg_row[1].strip('=') + version_list.append((pkg_row[0], Version(pkg_row[1]))) for pkg in pkgs: From c6495347622637df85e2e5e1b9384e42a8a4cae9 Mon Sep 17 00:00:00 2001 From: Martin Pecka Date: Thu, 12 Jan 2023 05:09:26 +0100 Subject: [PATCH 3/3] PipInstaller: More robust version string parsing. Signed-off-by: Martin Pecka --- src/rosdep2/platforms/pip.py | 38 +++++++++++++++++++++++++++++------- stdeb.cfg | 4 ++-- test/test_rosdep_pip.py | 15 ++++++++++++++ 3 files changed, 48 insertions(+), 9 deletions(-) diff --git a/src/rosdep2/platforms/pip.py b/src/rosdep2/platforms/pip.py index 89b0d3dcd..ed52ba58a 100644 --- a/src/rosdep2/platforms/pip.py +++ b/src/rosdep2/platforms/pip.py @@ -35,7 +35,7 @@ import sys from packaging.requirements import Requirement -from packaging.version import Version +from packaging.version import InvalidVersion, parse from ..core import InstallFailed from ..installers import PackageManagerInstaller from ..shell_utils import read_stdout @@ -79,6 +79,20 @@ def is_cmd_available(cmd): return False +def parse_version(version_str): + """ + Given a textual representation of a Python package version, return its parsed representation. + + :param version_str: The textual representation of the version. + :return: The parsed representation of None if the version cannot be parsed. + :rtype: packaging.version.Version or packaging.version.LegacyVersion or None + """ + try: + return parse(version_str) + except InvalidVersion: + return None + + def pip_detect(pkgs, exec_fn=None): """ Given a list of package specifications, return the list of installed @@ -95,6 +109,7 @@ def pip_detect(pkgs, exec_fn=None): exec_fn = read_stdout fallback_to_pip_show = True pkg_list = exec_fn(pip_cmd + ['freeze']).split('\n') + pkg_list = [p for p in pkg_list if len(p) > 0] ret_list = [] version_list = [] @@ -102,18 +117,18 @@ def pip_detect(pkgs, exec_fn=None): for pkg in pkg_list: pkg_row = pkg.split('==') - + # Account for some unusual instances of === instead of == pkg_row[1] = pkg_row[1].strip('=') - version_list.append((pkg_row[0], Version(pkg_row[1]))) + version_list.append((pkg_row[0], parse_version(pkg_row[1]))) for pkg in pkgs: req_list.append(Requirement(pkg)) for req in req_list: for pkg in [ver for ver in version_list if ver[0] == req.name]: - if pkg[1] in req.specifier: + if pkg[1] is None or pkg[1] in req.specifier: ret_list.append(req.name) # Try to detect with the return code of `pip show`. @@ -135,10 +150,10 @@ def pip_detect(pkgs, exec_fn=None): # `pip show` detected it, check the version. show_split = output.split('\n') - for line in [l for l in show_split if l.startswith('Version:')]: - version = Version(line.strip().split()[1]) + for line in [s for s in show_split if s.startswith('Version:')]: + version = parse_version(line.strip().split()[1]) - if version in req.specifier: + if version is None or version in req.specifier: # version matches, add it to the list ret_list.append(req.name) @@ -162,6 +177,15 @@ def get_version_strings(self): ] return version_strings + def get_packages_to_install(self, resolved, reinstall=False): + if reinstall: + return resolved + if not resolved: + return [] + else: + detected = self.detect_fn(resolved) + return [x for x in resolved if Requirement(x).name not in detected] + def get_install_command(self, resolved, interactive=True, reinstall=False, quiet=False): pip_cmd = get_pip_command() if not pip_cmd: diff --git a/stdeb.cfg b/stdeb.cfg index 1c906a33f..9bdaf77ef 100644 --- a/stdeb.cfg +++ b/stdeb.cfg @@ -17,8 +17,8 @@ X-Python3-Version: >= 3.6 Setup-Env-Vars: SKIP_PYTHON_MODULES=1 [rosdep_modules] -Depends: ca-certificates, python-rospkg-modules (>= 1.4.0), python-yaml, python-catkin-pkg-modules (>= 0.4.0), python-rosdistro-modules (>= 0.7.5), python-setuptools, sudo -Depends3: ca-certificates, python3-rospkg-modules (>= 1.4.0), python3-yaml, python3-catkin-pkg-modules (>= 0.4.0), python3-rosdistro-modules (>= 0.7.5), python3-setuptools, sudo +Depends: ca-certificates, python-rospkg-modules (>= 1.4.0), python-yaml, python-catkin-pkg-modules (>= 0.4.0), python-rosdistro-modules (>= 0.7.5), python-setuptools, python-packaging, sudo +Depends3: ca-certificates, python3-rospkg-modules (>= 1.4.0), python3-yaml, python3-catkin-pkg-modules (>= 0.4.0), python3-rosdistro-modules (>= 0.7.5), python3-setuptools, python3-packaging, sudo Conflicts: python-rosdep (<< 0.18.0), python-rosdep2 Conflicts3: python3-rosdep (<< 0.18.0), python3-rosdep2 Replaces: python-rosdep (<< 0.18.0) diff --git a/test/test_rosdep_pip.py b/test/test_rosdep_pip.py index 34840ce1c..36970e978 100644 --- a/test/test_rosdep_pip.py +++ b/test/test_rosdep_pip.py @@ -62,6 +62,21 @@ def test_pip_detect(): val = pip_detect(['paramiko', 'fakito', 'pycrypto'], exec_fn=m) assert val == ['paramiko', 'pycrypto'], val + val = pip_detect(['paramiko', 'Brlapi==0.5.4', 'pycrypto'], exec_fn=m) + assert val == ['paramiko', 'Brlapi', 'pycrypto'], val + + val = pip_detect(['paramiko', 'Brlapi==0.5.5', 'pycrypto'], exec_fn=m) + assert val == ['paramiko', 'pycrypto'], val + + val = pip_detect(['Brlapi>0.5.*'], exec_fn=m) + assert val == ['Brlapi'], val + + val = pip_detect(['Brlapi>0.5.*,<0.6'], exec_fn=m) + assert val == ['Brlapi'], val + + val = pip_detect(['Brlapi>0.5.*,<0.5.4'], exec_fn=m) + assert val == [], val + def test_PipInstaller_get_depends(): # make sure PipInstaller supports depends