From 50940ebe6dd39351d8995af4128127d19ab6c319 Mon Sep 17 00:00:00 2001 From: Julien Baudon Date: Wed, 13 Nov 2024 11:34:18 +0100 Subject: [PATCH] #69 add apt proxy repo support --- README.md | 18 +++++- allowlists/apt.allowlist | 13 ++++ entrypoint.sh | 9 +-- integration_tests/Dockerfile | 1 + integration_tests/sources.list | 1 + nexus_allowlist/__about__.py | 2 +- nexus_allowlist/actions.py | 111 +++++++++++++++++++++++++++++---- nexus_allowlist/cli.py | 13 ++-- nexus_allowlist/nexus.py | 7 ++- nexus_allowlist/settings.py | 6 ++ 10 files changed, 157 insertions(+), 24 deletions(-) create mode 100644 allowlists/apt.allowlist create mode 100644 integration_tests/sources.list create mode 100644 nexus_allowlist/settings.py diff --git a/README.md b/README.md index 3e1c0ff..c60a2b4 100644 --- a/README.md +++ b/README.md @@ -26,7 +26,7 @@ Check and, if you would like, change the following environment variables for the | NEXUS_PATH | [Context path](https://help.sonatype.com/en/configuring-the-runtime-environment.html#changing-the-context-path) of Nexus OSS. Only used if the Nexus is hosted behind a reverse proxy with a URL like `https://your_url.domain/nexus/`. If not defined, the base URI remains `/`. | | ENTR_FALLBACK | If defined, don't use `entr` to check for allowlist updates (this will be less reactive but we have found `entr` to not work in some situations) | -Example allowlist files are included in the repository for [PyPI](allowlists/pypi.allowlist) and [CRAN](allowlists/cran.allowlist). +Example allowlist files are included in the repository for [PyPI](allowlists/pypi.allowlist), [CRAN](allowlists/cran.allowlist) and [APT](allowlists/apt.allowlist). The PyPI allowlist includes numpy, pandas, matplotlib and their dependencies. The CRAN allowlist includes cli and data.table You can add more packages by writing the package names, one per line, in the allowlist files. @@ -96,6 +96,22 @@ For example, - `install.packages("data.table")` should succeed - `install.packages("ggplot2")` should fail +#### APT + +You can edit '/etc/apt/sources.list' to use the Nexus APT proxy. + +For example + +``` +deb http://localhost:8080/repository/apt-proxy bookworm main +``` + +You should now only be able to install packages from the allowlist. +For example, + +- `sudo apt install libcurl4-openssl-dev` should succeed +- `sudo apt install tcpdump` should fail + ## Contributors ✨ Thanks goes to these wonderful people ([emoji key](https://allcontributors.org/docs/en/emoji-key)): diff --git a/allowlists/apt.allowlist b/allowlists/apt.allowlist new file mode 100644 index 0000000..76a0965 --- /dev/null +++ b/allowlists/apt.allowlist @@ -0,0 +1,13 @@ +r-recommended +r-cran-matrixmodels +libcurl4-openssl-dev +libv8-dev +libxml2-dev +cmake +libfontconfig1-dev +libharfbuzz-dev +libfribidi-dev +libfreetype6-dev +libpng-dev +libtiff5-dev +libjpeg-dev \ No newline at end of file diff --git a/entrypoint.sh b/entrypoint.sh index 3cea716..fe06c87 100755 --- a/entrypoint.sh +++ b/entrypoint.sh @@ -4,6 +4,7 @@ export NEXUS_DATA_DIR=/nexus-data export ALLOWLIST_DIR=/allowlists export PYPI_ALLOWLIST="$ALLOWLIST_DIR"/pypi.allowlist export CRAN_ALLOWLIST="$ALLOWLIST_DIR"/cran.allowlist +export APT_ALLOWLIST="$ALLOWLIST_DIR"/apt.allowlist timestamp() { date -Is @@ -37,7 +38,7 @@ nexus-allowlist --version if [ -f "$NEXUS_DATA_DIR/admin.password" ]; then echo "$(timestamp) Initial password file present, running initial configuration" nexus-allowlist --admin-password "$NEXUS_ADMIN_PASSWORD" --nexus-host "$NEXUS_HOST" --nexus-path "$NEXUS_PATH" --nexus-port "$NEXUS_PORT" change-initial-password --path "$NEXUS_DATA_DIR" - nexus-allowlist --admin-password "$NEXUS_ADMIN_PASSWORD" --nexus-host "$NEXUS_HOST" --nexus-path "$NEXUS_PATH" --nexus-port "$NEXUS_PORT" initial-configuration --packages "$NEXUS_PACKAGES" --pypi-package-file "$ALLOWLIST_DIR/pypi.allowlist" --cran-package-file "$ALLOWLIST_DIR/cran.allowlist" + nexus-allowlist --admin-password "$NEXUS_ADMIN_PASSWORD" --nexus-host "$NEXUS_HOST" --nexus-path "$NEXUS_PATH" --nexus-port "$NEXUS_PORT" initial-configuration --packages "$NEXUS_PACKAGES" --pypi-package-file "$PYPI_ALLOWLIST" --cran-package-file "$CRAN_ALLOWLIST" --apt-package-file "$APT_ALLOWLIST" else echo "$(timestamp) No initial password file found, skipping initial configuration" fi @@ -51,13 +52,13 @@ fi if [ -n "$ENTR_FALLBACK" ]; then echo "$(timestamp) Using fallback file monitoring" # Run allowlist configuration now - nexus-allowlist --admin-password "$NEXUS_ADMIN_PASSWORD" --nexus-host "$NEXUS_HOST" --nexus-path "$NEXUS_PATH" --nexus-port "$NEXUS_PORT" update-allowlists --packages "$NEXUS_PACKAGES" --pypi-package-file "$PYPI_ALLOWLIST" --cran-package-file "$CRAN_ALLOWLIST" + nexus-allowlist --admin-password "$NEXUS_ADMIN_PASSWORD" --nexus-host "$NEXUS_HOST" --nexus-path "$NEXUS_PATH" --nexus-port "$NEXUS_PORT" update-allowlists --packages "$NEXUS_PACKAGES" --pypi-package-file "$PYPI_ALLOWLIST" --cran-package-file "$CRAN_ALLOWLIST" --apt-package-file "$APT_ALLOWLIST" # Periodically check for modification of allowlist files and run configuration again when they are hash=$(hashes) while true; do new_hash=$(hashes) if [ "$hash" != "$new_hash" ]; then - nexus-allowlist --admin-password "$NEXUS_ADMIN_PASSWORD" --nexus-host "$NEXUS_HOST" --nexus-path "$NEXUS_PATH" --nexus-port "$NEXUS_PORT" update-allowlists --packages "$NEXUS_PACKAGES" --pypi-package-file "$PYPI_ALLOWLIST" --cran-package-file "$CRAN_ALLOWLIST" + nexus-allowlist --admin-password "$NEXUS_ADMIN_PASSWORD" --nexus-host "$NEXUS_HOST" --nexus-path "$NEXUS_PATH" --nexus-port "$NEXUS_PORT" update-allowlists --packages "$NEXUS_PACKAGES" --pypi-package-file "$PYPI_ALLOWLIST" --cran-package-file "$CRAN_ALLOWLIST" --apt-package-file "$APT_ALLOWLIST" hash=$new_hash fi sleep 5 @@ -65,5 +66,5 @@ if [ -n "$ENTR_FALLBACK" ]; then else echo "$(timestamp) Using entr for file monitoring" # Run allowlist configuration now, and again whenever allowlist files are modified - find "$ALLOWLIST_DIR"/*.allowlist | entr -n nexus-allowlist --admin-password "$NEXUS_ADMIN_PASSWORD" --nexus-host "$NEXUS_HOST" --nexus-path "$NEXUS_PATH" --nexus-port "$NEXUS_PORT" update-allowlists --packages "$NEXUS_PACKAGES" --pypi-package-file "$PYPI_ALLOWLIST" --cran-package-file "$CRAN_ALLOWLIST" + find "$ALLOWLIST_DIR"/*.allowlist | entr -n nexus-allowlist --admin-password "$NEXUS_ADMIN_PASSWORD" --nexus-host "$NEXUS_HOST" --nexus-path "$NEXUS_PATH" --nexus-port "$NEXUS_PORT" update-allowlists --packages "$NEXUS_PACKAGES" --pypi-package-file "$PYPI_ALLOWLIST" --cran-package-file "$CRAN_ALLOWLIST" --apt-package-file "$APT_ALLOWLIST" fi diff --git a/integration_tests/Dockerfile b/integration_tests/Dockerfile index 69190bd..1502f89 100644 --- a/integration_tests/Dockerfile +++ b/integration_tests/Dockerfile @@ -4,3 +4,4 @@ RUN apk add --no-cache --update python3 py3-pip R RUN mkdir -p /root/.config/pip COPY pip.conf /root/.config/pip/pip.conf COPY Rprofile /root/.Rprofile +COPY sources.list /etc/apt/sources.list \ No newline at end of file diff --git a/integration_tests/sources.list b/integration_tests/sources.list new file mode 100644 index 0000000..3a61f46 --- /dev/null +++ b/integration_tests/sources.list @@ -0,0 +1 @@ +deb http://localhost:8080/repository/apt-proxy bookworm main \ No newline at end of file diff --git a/nexus_allowlist/__about__.py b/nexus_allowlist/__about__.py index 2d81ab7..fb6e1af 100644 --- a/nexus_allowlist/__about__.py +++ b/nexus_allowlist/__about__.py @@ -1 +1 @@ -__version__ = "v0.11.0" +__version__ = "v0.12.0" diff --git a/nexus_allowlist/actions.py b/nexus_allowlist/actions.py index 40cd349..bbf595d 100644 --- a/nexus_allowlist/actions.py +++ b/nexus_allowlist/actions.py @@ -4,6 +4,12 @@ from pathlib import Path from nexus_allowlist.nexus import NexusAPI, RepositoryType +from nexus_allowlist.settings import ( + APT_DISTRO, + APT_REMOTE_URL, + CRAN_REMOTE_URL, + PYPI_REMOTE_URL, +) @dataclass @@ -17,12 +23,17 @@ class Repository: "pypi_proxy": Repository( repo_type=RepositoryType.PYPI, name="pypi-proxy", - remote_url="https://pypi.org/", + remote_url=PYPI_REMOTE_URL, ), "cran_proxy": Repository( repo_type=RepositoryType.CRAN, name="cran-proxy", - remote_url="https://cran.r-project.org/", + remote_url=CRAN_REMOTE_URL, + ), + "apt_proxy": Repository( + repo_type=RepositoryType.APT, + name="apt-proxy", + remote_url=APT_REMOTE_URL, ), } @@ -37,28 +48,33 @@ def check_package_files(args: argparse.Namespace) -> None: raise: Exception: if any declared allowlist file does not exist """ - for package_file in [args.pypi_package_file, args.cran_package_file]: + for package_file in [ + args.pypi_package_file, + args.cran_package_file, + args.apt_package_file, + ]: if package_file and not package_file.is_file(): msg = f"Package allowlist file {package_file} does not exist" raise Exception(msg) def get_allowlists( - pypi_package_file: Path, cran_package_file: Path -) -> tuple[list[str], list[str]]: + pypi_package_file: Path, cran_package_file: Path, apt_package_file: Path +) -> tuple[list[str], list[str], list[str]]: """ - Create allowlists for PyPI and CRAN packages + Create allowlists for PyPI, CRAN and APT packages Args: pypi_package_file: Path to the PyPI allowlist file or None cran_package_file: Path to the CRAN allowlist file or None Returns: - A tuple of the PyPI and CRAN allowlists (in that order). The lists are + A tuple of the PyPI, CRAN and APT allowlists (in that order). The lists are [] if the corresponding package file argument was None """ pypi_allowlist = [] cran_allowlist = [] + apt_allowlist = [] if pypi_package_file: pypi_allowlist = get_allowlist(pypi_package_file, repo_type=RepositoryType.PYPI) @@ -66,7 +82,10 @@ def get_allowlists( if cran_package_file: cran_allowlist = get_allowlist(cran_package_file, repo_type=RepositoryType.CRAN) - return (pypi_allowlist, cran_allowlist) + if apt_package_file: + apt_allowlist = get_allowlist(apt_package_file, repo_type=RepositoryType.APT) + + return (pypi_allowlist, cran_allowlist, apt_allowlist) def get_allowlist(allowlist_path: Path, repo_type: RepositoryType) -> list[str]: @@ -83,9 +102,9 @@ def get_allowlist(allowlist_path: Path, repo_type: RepositoryType) -> list[str]: allowlist = [] with open(allowlist_path) as allowlist_file: # Sanitise package names - # - convert to lower case if the package is on PyPI. Leave alone on CRAN to - # prevent issues with case-sensitivity - for PyPI replace strings of '.', '_' - # or '-' with '-' + # - convert to lower case if the package is on PyPI or APT. Leave alone on CRAN + # to prevent issues with case-sensitivity - for PyPI replace strings of '.', + # '_' or '-' with '-' # https://packaging.python.org/en/latest/guides/distributing-packages-using-setuptools/#name # - remove any blank entries, which act as a wildcard that would allow any # package @@ -94,6 +113,8 @@ def get_allowlist(allowlist_path: Path, repo_type: RepositoryType) -> list[str]: match repo_type: case RepositoryType.CRAN: package_name_parsed = package_name.strip() + case RepositoryType.APT: + package_name_parsed = package_name.lower().strip() case RepositoryType.PYPI: package_name_parsed = pypi_replace_characters.sub( "-", package_name.lower().strip() @@ -104,7 +125,7 @@ def get_allowlist(allowlist_path: Path, repo_type: RepositoryType) -> list[str]: def recreate_repositories(nexus_api: NexusAPI) -> None: """ - Create PyPI and CRAN proxy repositories in an idempotent manner + Create PyPI, CRAN and APT proxy repositories in an idempotent manner Args: nexus_api: NexusAPI object @@ -125,6 +146,7 @@ def recreate_privileges( nexus_api: NexusAPI, pypi_allowlist: list[str], cran_allowlist: list[str], + apt_allowlist: list[str], ) -> list[str]: """ Create content selectors and content selector privileges based on the @@ -134,6 +156,7 @@ def recreate_privileges( nexus_api: NexusAPI object pypi_allowlist: List of allowed PyPI packages cran_allowlist: List of allowed CRAN packages + apt_allowlist: List of allowed APT packages Returns: List of the names of all content selector privileges @@ -148,6 +171,7 @@ def recreate_privileges( pypi_privilege_names = [] cran_privilege_names = [] + apt_privilege_names = [] # Content selector and privilege for PyPI 'simple' path, used to search for # packages @@ -185,6 +209,44 @@ def recreate_privileges( ) cran_privilege_names.append(privilege_name) + # Content selector and privilege for APT 'Packages.gz' file which contains an + # metadata for all archived packages + privilege_name = create_content_selector_and_privilege( + nexus_api, + name="apt-packages", + description="Allow access to 'Packages.gz' file in APT repository", + expression=f'format == "apt" and path=~"^/dists/{APT_DISTRO}/.*/Packages.gz"', + repo_type=_NEXUS_REPOSITORIES["apt_proxy"].repo_type, + repo=_NEXUS_REPOSITORIES["apt_proxy"].name, + ) + apt_privilege_names.append(privilege_name) + + # Content selector and privilege for APT 'InRelease' file which contains an + # metadata about the APT distribution + privilege_name = create_content_selector_and_privilege( + nexus_api, + name="inrelease", + description="Allow access to 'InRelease' file in APT repository", + expression=f'format == "apt" and path=="/dists/{APT_DISTRO}/InRelease"', + repo_type=_NEXUS_REPOSITORIES["apt_proxy"].repo_type, + repo=_NEXUS_REPOSITORIES["apt_proxy"].name, + ) + apt_privilege_names.append(privilege_name) + + # Content selector and privilege for APT 'Translation-*' files which contains an + # metadata about the APT distribution + privilege_name = create_content_selector_and_privilege( + nexus_api, + name="apt-translation", + description="Allow access to 'Translation-*' file in APT repository", + expression=( + f'format == "apt" and path=~"^/dists/{APT_DISTRO}/.*/Translation-.*"' + ), + repo_type=_NEXUS_REPOSITORIES["apt_proxy"].repo_type, + repo=_NEXUS_REPOSITORIES["apt_proxy"].name, + ) + apt_privilege_names.append(privilege_name) + # Create content selectors and privileges for packages according to the # package setting if packages == "all": @@ -209,6 +271,17 @@ def recreate_privileges( repo=_NEXUS_REPOSITORIES["cran_proxy"].name, ) cran_privilege_names.append(privilege_name) + + # Allow all APT packages + privilege_name = create_content_selector_and_privilege( + nexus_api, + name="apt-all", + description="Allow access to all APT packages", + expression='format == "apt" and path=^"/pool/"', + repo_type=_NEXUS_REPOSITORIES["apt_proxy"].repo_type, + repo=_NEXUS_REPOSITORIES["apt_proxy"].name, + ) + apt_privilege_names.append(privilege_name) elif packages == "selected": # Allow selected PyPI packages for package in pypi_allowlist: @@ -238,7 +311,19 @@ def recreate_privileges( ) cran_privilege_names.append(privilege_name) - return pypi_privilege_names + cran_privilege_names + # Allow selected APT packages + for package in apt_allowlist: + privilege_name = create_content_selector_and_privilege( + nexus_api, + name=f"apt-{package}", + description=f"Allow access to {packages} APT package", + expression=f'format == "apt" and path=~"^/pool/.*/{package}.*"', + repo_type=_NEXUS_REPOSITORIES["apt_proxy"].repo_type, + repo=_NEXUS_REPOSITORIES["apt_proxy"].name, + ) + apt_privilege_names.append(privilege_name) + + return pypi_privilege_names + cran_privilege_names + apt_privilege_names def create_content_selector_and_privilege( diff --git a/nexus_allowlist/cli.py b/nexus_allowlist/cli.py index bad5cfe..abab00c 100644 --- a/nexus_allowlist/cli.py +++ b/nexus_allowlist/cli.py @@ -76,6 +76,11 @@ def main() -> None: "Path of the file of allowed CRAN packages, ignored when PACKAGES is all" ), ) + packages_parser.add_argument( + "--apt-package-file", + type=Path, + help="Path of the file of allowed APT packages, ignored when PACKAGES is all", + ) subparsers = parser.add_subparsers(title="subcommands", required=True) @@ -168,7 +173,7 @@ def initial_configuration(args: argparse.Namespace) -> None: This includes: - Deleting all respositories - - Creating CRAN and PyPI proxies + - Creating CRAN, APT and PyPI proxies - Deleting all content selectors and content selector privileges - Deleting all non-default roles - Creating a role @@ -234,14 +239,14 @@ def update_allow_lists(args: argparse.Namespace) -> None: ) # Parse allowlists - pypi_allowlist, cran_allowlist = actions.get_allowlists( - args.pypi_package_file, args.cran_package_file + pypi_allowlist, cran_allowlist, apt_allowlist = actions.get_allowlists( + args.pypi_package_file, args.cran_package_file, args.apt_package_file ) # Recreate all content selectors and associated privileges according to the # allowlists privileges = actions.recreate_privileges( - args.packages, nexus_api, pypi_allowlist, cran_allowlist + args.packages, nexus_api, pypi_allowlist, cran_allowlist, apt_allowlist ) # Grant privileges to the nexus allowlist role diff --git a/nexus_allowlist/nexus.py b/nexus_allowlist/nexus.py index a32fb2f..9045052 100644 --- a/nexus_allowlist/nexus.py +++ b/nexus_allowlist/nexus.py @@ -4,6 +4,8 @@ import requests +from nexus_allowlist.settings import APT_DISTRO + _REQUEST_TIMEOUT = 10 @@ -21,6 +23,7 @@ class ResponseCode(Enum): class RepositoryType(Enum): PYPI = "pypi" CRAN = "r" + APT = "apt" class NexusAPI: @@ -96,7 +99,7 @@ def create_proxy_repository( self, repo_type: RepositoryType, name: str, remote_url: str ) -> None: """ - Create a proxy repository. Currently supports PyPI and R formats + Create a proxy repository. Currently supports PyPI, R and APT formats Args: repo_type: Type of repository @@ -123,6 +126,8 @@ def create_proxy_repository( } payload["name"] = name payload["proxy"]["remoteUrl"] = remote_url + if repo_type == RepositoryType.APT: + payload["apt"] = {"distribution": APT_DISTRO, "flat": False} logging.info(f"Creating {repo_type.value} repository: {name}") response = requests.post( diff --git a/nexus_allowlist/settings.py b/nexus_allowlist/settings.py new file mode 100644 index 0000000..de6940f --- /dev/null +++ b/nexus_allowlist/settings.py @@ -0,0 +1,6 @@ +import os + +PYPI_REMOTE_URL = os.getenv("PYPI_REMOTE_URL", "https://pypi.org/") +CRAN_REMOTE_URL = os.getenv("CRAN_REMOTE_URL", "https://cran.r-project.org/") +APT_REMOTE_URL = os.getenv("APT_REMOTE_URL", "http://deb.debian.org/debian") +APT_DISTRO = os.getenv("APT_DISTRO", "bookworm")