From 2ac789fe8ac0db98521f716f429113be42dcdbc9 Mon Sep 17 00:00:00 2001 From: Florian Maas Date: Thu, 4 Apr 2024 14:10:49 +0200 Subject: [PATCH] Automatically use `requirements.in` if the project uses a `requirements.txt` + `requirements.in`-setup (#641) --- docs/usage.md | 6 +- pdm.lock | 6 +- python/deptry/cli.py | 15 +-- python/deptry/core.py | 2 + python/deptry/dependency_getter/builder.py | 25 ++++- .../pyproject.toml | 5 + .../requirements-dev.txt | 1 + .../requirements.in | 4 + .../requirements.txt | 7 ++ .../project_with_requirements_in/src/main.py | 8 ++ .../src/notebook.ipynb | 37 +++++++ .../cli/test_cli_requirements_in.py | 100 ++++++++++++++++++ tests/functional/utils.py | 1 + tests/unit/dependency_getter/test_builder.py | 32 ++++++ tests/unit/deprecate/test_requirements_txt.py | 1 + tests/unit/test_core.py | 1 + 16 files changed, 234 insertions(+), 17 deletions(-) create mode 100644 tests/data/project_with_requirements_in/pyproject.toml create mode 100644 tests/data/project_with_requirements_in/requirements-dev.txt create mode 100644 tests/data/project_with_requirements_in/requirements.in create mode 100644 tests/data/project_with_requirements_in/requirements.txt create mode 100644 tests/data/project_with_requirements_in/src/main.py create mode 100644 tests/data/project_with_requirements_in/src/notebook.ipynb create mode 100644 tests/functional/cli/test_cli_requirements_in.py diff --git a/docs/usage.md b/docs/usage.md index 4d0e1971..e794615e 100644 --- a/docs/usage.md +++ b/docs/usage.md @@ -31,9 +31,9 @@ To determine the project's dependencies, _deptry_ will scan the directory it is 3. If a `pyproject.toml` file with a `[project]` section is found, _deptry_ will assume it uses [PEP 621](https://peps.python.org/pep-0621/) for dependency specification and extract: - dependencies from `[project.dependencies]` and `[project.optional-dependencies]`. - development dependencies from the groups under `[project.optional-dependencies]` passed via the [`--pep621-dev-dependency-groups`](#pep-621-dev-dependency-groups) argument. -4. If a `requirements.txt` file is found, _deptry_ will extract: - - dependencies from it - - development dependencies from `dev-dependencies.txt` and `dependencies-dev.txt`, if any exist +4. If a `requirements.in` or `requirements.txt` file is found, _deptry_ will: + - extract dependencies from that file. + - extract development dependencies from `dev-dependencies.txt` and `dependencies-dev.txt`, if any exist _deptry_ can be configured to look for `pip` requirements files with other names or in other directories. See [Requirements files](#requirements-files) and [Requirements files dev](#requirements-files-dev). diff --git a/pdm.lock b/pdm.lock index 0aa83b56..a42ec149 100644 --- a/pdm.lock +++ b/pdm.lock @@ -318,14 +318,14 @@ files = [ [[package]] name = "filelock" -version = "3.13.1" +version = "3.13.3" requires_python = ">=3.8" summary = "A platform independent file lock." groups = ["dev"] marker = "python_version >= \"3.9\"" files = [ - {file = "filelock-3.13.1-py3-none-any.whl", hash = "sha256:57dbda9b35157b05fb3e58ee91448612eb674172fab98ee235ccb0b5bee19a1c"}, - {file = "filelock-3.13.1.tar.gz", hash = "sha256:521f5f56c50f8426f5e03ad3b281b490a87ef15bc6c526f168290f0c7148d44e"}, + {file = "filelock-3.13.3-py3-none-any.whl", hash = "sha256:5ffa845303983e7a0b7ae17636509bc97997d58afeafa72fb141a17b152284cb"}, + {file = "filelock-3.13.3.tar.gz", hash = "sha256:a79895a25bbefdf55d1a2a0a80968f7dbb28edcd6d4234a0afb3f37ecde4b546"}, ] [[package]] diff --git a/python/deptry/cli.py b/python/deptry/cli.py index 0ea607e8..f8dfb33f 100644 --- a/python/deptry/cli.py +++ b/python/deptry/cli.py @@ -26,6 +26,8 @@ DEFAULT_EXCLUDE = ("venv", r"\.venv", r"\.direnv", "tests", r"\.git", r"setup\.py") +DEFAULT_REQUIREMENTS_FILES = ("requirements.txt",) + class CommaSeparatedTupleParamType(click.ParamType): """ @@ -159,8 +161,7 @@ def display_deptry_version(ctx: click.Context, _param: click.Parameter, value: b help=f"""A regular expression for directories or files in which .py files should not be scanned for imports to determine if there are dependency issues. Can be used multiple times by specifying the argument multiple times. re.match() is used to match the expressions, which by default checks for a match only at the beginning of a string. For example: `deptry . -e ".*/foo/" -e bar"` Note that this overwrites the defaults. - [default: {", ".join(DEFAULT_EXCLUDE)} - """, + [default: {", ".join(DEFAULT_EXCLUDE)}]""", ) @click.option( "--extend-exclude", @@ -196,10 +197,9 @@ def display_deptry_version(ctx: click.Context, _param: click.Parameter, value: b "--requirements-files", "-rf", type=COMMA_SEPARATED_TUPLE, - help=""".txt files to scan for dependencies. If a file called pyproject.toml with a [tool.poetry.dependencies] or [project] section is found, this argument is ignored - and the dependencies are extracted from the pyproject.toml file instead. Can be multiple e.g. `deptry . --requirements-txt req/prod.txt,req/extra.txt`""", - default=("requirements.txt",), - show_default=True, + help=f""".txt files to scan for dependencies. If a file called pyproject.toml with a [tool.poetry.dependencies] or [project] section is found, this argument is ignored + and the dependencies are extracted from the pyproject.toml file instead. Can be multiple e.g. `deptry . --requirements-txt req/prod.txt,req/extra.txt` + [default: {", ".join(DEFAULT_REQUIREMENTS_FILES)}]""", ) @click.option( "--requirements-files-dev", @@ -292,7 +292,8 @@ def deptry( ignore_notebooks=ignore_notebooks, ignore=ignore, per_rule_ignores=per_rule_ignores, - requirements_files=requirements_txt or requirements_files, + requirements_files=(requirements_txt or requirements_files) or DEFAULT_REQUIREMENTS_FILES, + using_default_requirements_files=not (requirements_txt or requirements_files), requirements_files_dev=requirements_txt_dev or requirements_files_dev, known_first_party=known_first_party, json_output=json_output, diff --git a/python/deptry/core.py b/python/deptry/core.py index 71cd7960..265d4fae 100644 --- a/python/deptry/core.py +++ b/python/deptry/core.py @@ -34,6 +34,7 @@ class Core: using_default_exclude: bool ignore_notebooks: bool requirements_files: tuple[str, ...] + using_default_requirements_files: bool requirements_files_dev: tuple[str, ...] known_first_party: tuple[str, ...] json_output: str @@ -48,6 +49,7 @@ def run(self) -> None: self.package_module_name_map, self.pep621_dev_dependency_groups, self.requirements_files, + self.using_default_requirements_files, self.requirements_files_dev, ).build() diff --git a/python/deptry/dependency_getter/builder.py b/python/deptry/dependency_getter/builder.py index ae64f7dd..14fb16f4 100644 --- a/python/deptry/dependency_getter/builder.py +++ b/python/deptry/dependency_getter/builder.py @@ -32,6 +32,7 @@ class DependencyGetterBuilder: package_module_name_map: Mapping[str, tuple[str, ...]] = field(default_factory=dict) pep621_dev_dependency_groups: tuple[str, ...] = () requirements_files: tuple[str, ...] = () + using_default_requirements_files: bool = True requirements_files_dev: tuple[str, ...] = () def build(self) -> DependencyGetter: @@ -51,9 +52,10 @@ def build(self) -> DependencyGetter: self.config, self.package_module_name_map, self.pep621_dev_dependency_groups ) - if self._project_uses_requirements_files(): + check, requirements_files = self._project_uses_requirements_files() + if check: return RequirementsTxtDependencyGetter( - self.config, self.package_module_name_map, self.requirements_files, self.requirements_files_dev + self.config, self.package_module_name_map, requirements_files, self.requirements_files_dev ) raise DependencySpecificationNotFoundError(self.requirements_files) @@ -114,11 +116,26 @@ def _project_uses_pep_621(pyproject_toml: dict[str, Any]) -> bool: ) return False - def _project_uses_requirements_files(self) -> bool: + def _project_uses_requirements_files(self) -> tuple[bool, tuple[str, ...]]: + """ + Tools like `pip-tools` and `uv` work with a setup in which a `requirements.in` is compiled into a `requirements.txt`, which then + contains pinned versions for all transitive dependencies. If the user did not explicitly specify the argument `requirements-files`, + but there is a `requirements.in` present, it is highly likely that the user wants to use the `requirements.in` file so we set + `requirements-files` to that instead. + """ + if self.using_default_requirements_files and Path("requirements.in").is_file(): + logging.info( + "Detected a 'requirements.in' file in the project and no 'requirements-files' were explicitly specified. " + "Automatically using 'requirements.in' as the source for the project's dependencies. To specify a different source for " + "the project's dependencies, use the '--requirements-files' option." + ) + return True, ("requirements.in",) + check = any(Path(requirements_files).is_file() for requirements_files in self.requirements_files) if check: logging.debug( "Dependency specification found in '%s'. Will use this to determine the project's dependencies.\n", ", ".join(self.requirements_files), ) - return check + return True, self.requirements_files + return False, () diff --git a/tests/data/project_with_requirements_in/pyproject.toml b/tests/data/project_with_requirements_in/pyproject.toml new file mode 100644 index 00000000..a42cf85c --- /dev/null +++ b/tests/data/project_with_requirements_in/pyproject.toml @@ -0,0 +1,5 @@ +[tool.black] +line-length = 120 + +[tool.deptry.per_rule_ignores] +DEP001 = ["toml"] diff --git a/tests/data/project_with_requirements_in/requirements-dev.txt b/tests/data/project_with_requirements_in/requirements-dev.txt new file mode 100644 index 00000000..b173f012 --- /dev/null +++ b/tests/data/project_with_requirements_in/requirements-dev.txt @@ -0,0 +1 @@ +black==22.6.0 diff --git a/tests/data/project_with_requirements_in/requirements.in b/tests/data/project_with_requirements_in/requirements.in new file mode 100644 index 00000000..e63b7fef --- /dev/null +++ b/tests/data/project_with_requirements_in/requirements.in @@ -0,0 +1,4 @@ +click==8.1.3 +isort==5.10.1 +urllib3 +pandas diff --git a/tests/data/project_with_requirements_in/requirements.txt b/tests/data/project_with_requirements_in/requirements.txt new file mode 100644 index 00000000..23b33f0e --- /dev/null +++ b/tests/data/project_with_requirements_in/requirements.txt @@ -0,0 +1,7 @@ +# Generated from requirements.in +click==8.1.3 +isort==5.10.1 +urllib3 +requests +pandas +numpy diff --git a/tests/data/project_with_requirements_in/src/main.py b/tests/data/project_with_requirements_in/src/main.py new file mode 100644 index 00000000..2ecc04af --- /dev/null +++ b/tests/data/project_with_requirements_in/src/main.py @@ -0,0 +1,8 @@ +from os import chdir, walk +from pathlib import Path + +import black +import click +import white as w +from urllib3 import contrib +import requests diff --git a/tests/data/project_with_requirements_in/src/notebook.ipynb b/tests/data/project_with_requirements_in/src/notebook.ipynb new file mode 100644 index 00000000..a51bdb9d --- /dev/null +++ b/tests/data/project_with_requirements_in/src/notebook.ipynb @@ -0,0 +1,37 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 2, + "id": "9f4924ec-2200-4801-9d49-d4833651cbc4", + "metadata": {}, + "outputs": [], + "source": [ + "import click\n", + "from urllib3 import contrib\n", + "import toml" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.9.11" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/tests/functional/cli/test_cli_requirements_in.py b/tests/functional/cli/test_cli_requirements_in.py new file mode 100644 index 00000000..8dbd0fca --- /dev/null +++ b/tests/functional/cli/test_cli_requirements_in.py @@ -0,0 +1,100 @@ +from __future__ import annotations + +import uuid +from pathlib import Path +from typing import TYPE_CHECKING + +import pytest + +from tests.functional.utils import Project +from tests.utils import get_issues_report + +if TYPE_CHECKING: + from tests.utils import PipVenvFactory + + +@pytest.mark.xdist_group(name=Project.REQUIREMENTS_IN) +def test_cli_single_requirements_files(pip_venv_factory: PipVenvFactory) -> None: + """ + in this case, deptry should recognize that there is a `requirements.in` in the project, and + use that as the source of the dependencies. + """ + with pip_venv_factory( + Project.REQUIREMENTS_IN, + install_command=("pip install -r requirements.txt -r requirements-dev.txt"), + ) as virtual_env: + issue_report = f"{uuid.uuid4()}.json" + result = virtual_env.run(f"deptry . -o {issue_report}") + + assert result.returncode == 1 + assert get_issues_report(Path(issue_report)) == [ + { + "error": {"code": "DEP002", "message": "'isort' defined as a dependency but not used in the codebase"}, + "module": "isort", + "location": {"file": str(Path("requirements.in")), "line": None, "column": None}, + }, + { + "error": {"code": "DEP002", "message": "'pandas' defined as a dependency but not used in the codebase"}, + "module": "pandas", + "location": {"file": str(Path("requirements.in")), "line": None, "column": None}, + }, + { + "error": {"code": "DEP004", "message": "'black' imported but declared as a dev dependency"}, + "module": "black", + "location": {"file": str(Path("src/main.py")), "line": 4, "column": 8}, + }, + { + "error": {"code": "DEP001", "message": "'white' imported but missing from the dependency definitions"}, + "module": "white", + "location": {"file": str(Path("src/main.py")), "line": 6, "column": 8}, + }, + { + "error": {"code": "DEP003", "message": "'requests' imported but it is a transitive dependency"}, + "module": "requests", + "location": {"file": str(Path("src/main.py")), "line": 8, "column": 8}, + }, + ] + + +@pytest.mark.xdist_group(name=Project.REQUIREMENTS_IN) +def test_cli_multiple_requirements_files(pip_venv_factory: PipVenvFactory) -> None: + """ + in this case, deptry recognizes that there is a `requirements.in` in the project, but the user + can overwrite that with '--requirements-files requirements.txt', so it still takes requirements.txt as the source + for the project's dependencies. + """ + with pip_venv_factory( + Project.REQUIREMENTS_IN, + install_command=("pip install -r requirements.txt -r requirements-dev.txt"), + ) as virtual_env: + issue_report = f"{uuid.uuid4()}.json" + result = virtual_env.run(f"deptry . --requirements-files requirements.txt -o {issue_report}") + + assert result.returncode == 1 + assert get_issues_report(Path(issue_report)) == [ + { + "error": {"code": "DEP002", "message": "'isort' defined as a dependency but not used in the codebase"}, + "module": "isort", + "location": {"file": str(Path("requirements.txt")), "line": None, "column": None}, + }, + { + "error": {"code": "DEP002", "message": "'pandas' defined as a dependency but not used in the codebase"}, + "module": "pandas", + "location": {"file": str(Path("requirements.txt")), "line": None, "column": None}, + }, + { + "error": {"code": "DEP002", "message": "'numpy' defined as a dependency but not used in the codebase"}, + "module": "numpy", + "location": {"file": str(Path("requirements.txt")), "line": None, "column": None}, + }, + { + "error": {"code": "DEP004", "message": "'black' imported but declared as a dev dependency"}, + "module": "black", + "location": {"file": str(Path("src/main.py")), "line": 4, "column": 8}, + }, + { + "error": {"code": "DEP001", "message": "'white' imported but missing from the dependency definitions"}, + "module": "white", + "location": {"file": str(Path("src/main.py")), "line": 6, "column": 8}, + }, + ] diff --git a/tests/functional/utils.py b/tests/functional/utils.py index b7c9a9db..08853829 100644 --- a/tests/functional/utils.py +++ b/tests/functional/utils.py @@ -15,6 +15,7 @@ class Project(str, Enum): POETRY = "project_with_poetry" PYPROJECT_DIFFERENT_DIRECTORY = "project_with_pyproject_different_directory" REQUIREMENTS_TXT = "project_with_requirements_txt" + REQUIREMENTS_IN = "project_with_requirements_in" SRC_DIRECTORY = "project_with_src_directory" def __str__(self) -> str: diff --git a/tests/unit/dependency_getter/test_builder.py b/tests/unit/dependency_getter/test_builder.py index e41a9d2a..5ea7ead4 100644 --- a/tests/unit/dependency_getter/test_builder.py +++ b/tests/unit/dependency_getter/test_builder.py @@ -3,6 +3,7 @@ import logging import re from pathlib import Path +from typing import TYPE_CHECKING import pytest @@ -14,6 +15,9 @@ from deptry.exceptions import DependencySpecificationNotFoundError from tests.utils import run_within_dir +if TYPE_CHECKING: + from _pytest.logging import LogCaptureFixture + def test_poetry(tmp_path: Path) -> None: with run_within_dir(tmp_path): @@ -131,3 +135,31 @@ def test_dependency_specification_not_found_raises_exception(tmp_path: Path, cap " dependencies." ), ] + + +def test_check_for_requirements_in_file_with_requirements_in(tmp_path: Path, caplog: LogCaptureFixture) -> None: + with run_within_dir(tmp_path): + # Setup: Create a requirements.in file in the temporary directory + requirements_in_path = Path("requirements.in") + requirements_in_path.touch() + requirements_txt_path = Path("requirements.txt") + requirements_txt_path.touch() + + # Use caplog to capture logging at the INFO level + with caplog.at_level(logging.INFO): + spec = DependencyGetterBuilder( + config=Path("pyproject.toml"), + requirements_files=("requirements.txt",), + using_default_requirements_files=True, + ).build() + + # Assert that requirements_files is updated correctly + assert spec.requirements_files == ("requirements.in",) # type: ignore[attr-defined] + + # Assert that the expected log message is present + expected_log = ( + "Detected a 'requirements.in' file in the project and no 'requirements-files' were explicitly specified. " + "Automatically using 'requirements.in' as the source for the project's dependencies. To specify a different source for " + "the project's dependencies, use the '--requirements-files' option." + ) + assert expected_log in caplog.text diff --git a/tests/unit/deprecate/test_requirements_txt.py b/tests/unit/deprecate/test_requirements_txt.py index 6b9920bb..8379aae2 100644 --- a/tests/unit/deprecate/test_requirements_txt.py +++ b/tests/unit/deprecate/test_requirements_txt.py @@ -24,6 +24,7 @@ "json_output": ANY, "package_module_name_map": ANY, "pep621_dev_dependency_groups": ANY, + "using_default_requirements_files": ANY, } diff --git a/tests/unit/test_core.py b/tests/unit/test_core.py index 4f71287f..07ac5bba 100644 --- a/tests/unit/test_core.py +++ b/tests/unit/test_core.py @@ -74,6 +74,7 @@ def test__get_local_modules( json_output="", package_module_name_map={}, pep621_dev_dependency_groups=(), + using_default_requirements_files=True, )._get_local_modules() == expected )