alan-turing-institute#69 add apt proxy repo support

README.md

@@ -26,7 +26,7 @@ Check and, if you would like, change the following environment variables for the
 | NEXUS_PATH             | [Context path](https://help.sonatype.com/en/configuring-the-runtime-environment.html#changing-the-context-path) of Nexus OSS. Only used if the Nexus is hosted behind a reverse proxy with a URL like `https://your_url.domain/nexus/`. If not defined, the base URI remains `/`.                                                                                                                              |
 | ENTR_FALLBACK          | If defined, don't use `entr` to check for allowlist updates (this will be less reactive but we have found `entr` to not work in some situations) |
 
-Example allowlist files are included in the repository for [PyPI](allowlists/pypi.allowlist) and [CRAN](allowlists/cran.allowlist).
+Example allowlist files are included in the repository for [PyPI](allowlists/pypi.allowlist), [CRAN](allowlists/cran.allowlist) and [APT](allowlists/apt.allowlist).
 The PyPI allowlist includes numpy, pandas, matplotlib and their dependencies.
 The CRAN allowlist includes cli and data.table
 You can add more packages by writing the package names, one per line, in the allowlist files.
@@ -96,6 +96,22 @@ For example,
 - `install.packages("data.table")` should succeed
 - `install.packages("ggplot2")` should fail
 
+#### APT
+
+You can edit '/etc/apt/sources.list' to use the Nexus APT proxy.
+
+For example
+
+```
+deb http://localhost:8080/repository/apt-proxy bookworm main
+```
+
+You should now only be able to install packages from the allowlist.
+For example,
+
+- `sudo apt install libcurl4-openssl-dev` should succeed
+- `sudo apt install tcpdump` should fail
+
 ## Contributors ✨
 
 Thanks goes to these wonderful people ([emoji key](https://allcontributors.org/docs/en/emoji-key)):

allowlists/apt.allowlist

@@ -0,0 +1,13 @@
+r-recommended
+r-cran-matrixmodels
+libcurl4-openssl-dev
+libv8-dev
+libxml2-dev
+cmake
+libfontconfig1-dev
+libharfbuzz-dev
+libfribidi-dev
+libfreetype6-dev
+libpng-dev
+libtiff5-dev
+libjpeg-dev

entrypoint.sh

@@ -4,6 +4,7 @@ export NEXUS_DATA_DIR=/nexus-data
 export ALLOWLIST_DIR=/allowlists
 export PYPI_ALLOWLIST="$ALLOWLIST_DIR"/pypi.allowlist
 export CRAN_ALLOWLIST="$ALLOWLIST_DIR"/cran.allowlist
+export APT_ALLOWLIST="$ALLOWLIST_DIR"/apt.allowlist
 
 timestamp() {
     date -Is
@@ -37,7 +38,7 @@ nexus-allowlist --version
 if [ -f "$NEXUS_DATA_DIR/admin.password" ]; then
     echo "$(timestamp) Initial password file present, running initial configuration"
     nexus-allowlist --admin-password "$NEXUS_ADMIN_PASSWORD" --nexus-host "$NEXUS_HOST" --nexus-path "$NEXUS_PATH" --nexus-port "$NEXUS_PORT" change-initial-password --path "$NEXUS_DATA_DIR"
-    nexus-allowlist --admin-password "$NEXUS_ADMIN_PASSWORD" --nexus-host "$NEXUS_HOST" --nexus-path "$NEXUS_PATH" --nexus-port "$NEXUS_PORT" initial-configuration --packages "$NEXUS_PACKAGES" --pypi-package-file "$ALLOWLIST_DIR/pypi.allowlist" --cran-package-file "$ALLOWLIST_DIR/cran.allowlist"
+    nexus-allowlist --admin-password "$NEXUS_ADMIN_PASSWORD" --nexus-host "$NEXUS_HOST" --nexus-path "$NEXUS_PATH" --nexus-port "$NEXUS_PORT" initial-configuration --packages "$NEXUS_PACKAGES" --pypi-package-file "$PYPI_ALLOWLIST" --cran-package-file "$CRAN_ALLOWLIST" --apt-package-file "$APT_ALLOWLIST"
 else
     echo "$(timestamp) No initial password file found, skipping initial configuration"
 fi
@@ -51,19 +52,19 @@ fi
 if [ -n "$ENTR_FALLBACK" ]; then
     echo "$(timestamp) Using fallback file monitoring"
     # Run allowlist configuration now
-    nexus-allowlist --admin-password "$NEXUS_ADMIN_PASSWORD" --nexus-host "$NEXUS_HOST" --nexus-path "$NEXUS_PATH" --nexus-port "$NEXUS_PORT" update-allowlists --packages "$NEXUS_PACKAGES" --pypi-package-file "$PYPI_ALLOWLIST" --cran-package-file "$CRAN_ALLOWLIST"
+    nexus-allowlist --admin-password "$NEXUS_ADMIN_PASSWORD" --nexus-host "$NEXUS_HOST" --nexus-path "$NEXUS_PATH" --nexus-port "$NEXUS_PORT" update-allowlists --packages "$NEXUS_PACKAGES" --pypi-package-file "$PYPI_ALLOWLIST" --cran-package-file "$CRAN_ALLOWLIST" --apt-package-file "$APT_ALLOWLIST"
     # Periodically check for modification of allowlist files and run configuration again when they are
     hash=$(hashes)
     while true; do
         new_hash=$(hashes)
         if [ "$hash" != "$new_hash" ]; then
-            nexus-allowlist --admin-password "$NEXUS_ADMIN_PASSWORD" --nexus-host "$NEXUS_HOST" --nexus-path "$NEXUS_PATH" --nexus-port "$NEXUS_PORT" update-allowlists --packages "$NEXUS_PACKAGES" --pypi-package-file "$PYPI_ALLOWLIST" --cran-package-file "$CRAN_ALLOWLIST"
+            nexus-allowlist --admin-password "$NEXUS_ADMIN_PASSWORD" --nexus-host "$NEXUS_HOST" --nexus-path "$NEXUS_PATH" --nexus-port "$NEXUS_PORT" update-allowlists --packages "$NEXUS_PACKAGES" --pypi-package-file "$PYPI_ALLOWLIST" --cran-package-file "$CRAN_ALLOWLIST" --apt-package-file "$APT_ALLOWLIST"
             hash=$new_hash
         fi
         sleep 5
     done
 else
     echo "$(timestamp) Using entr for file monitoring"
     # Run allowlist configuration now, and again whenever allowlist files are modified
-    find "$ALLOWLIST_DIR"/*.allowlist | entr -n nexus-allowlist --admin-password "$NEXUS_ADMIN_PASSWORD" --nexus-host "$NEXUS_HOST" --nexus-path "$NEXUS_PATH" --nexus-port "$NEXUS_PORT" update-allowlists --packages "$NEXUS_PACKAGES" --pypi-package-file "$PYPI_ALLOWLIST" --cran-package-file "$CRAN_ALLOWLIST"
+    find "$ALLOWLIST_DIR"/*.allowlist | entr -n nexus-allowlist --admin-password "$NEXUS_ADMIN_PASSWORD" --nexus-host "$NEXUS_HOST" --nexus-path "$NEXUS_PATH" --nexus-port "$NEXUS_PORT" update-allowlists --packages "$NEXUS_PACKAGES" --pypi-package-file "$PYPI_ALLOWLIST" --cran-package-file "$CRAN_ALLOWLIST" --apt-package-file "$APT_ALLOWLIST"
 fi

integration_tests/Dockerfile

@@ -4,3 +4,4 @@ RUN apk add --no-cache --update python3 py3-pip R
 RUN mkdir -p /root/.config/pip
 COPY pip.conf /root/.config/pip/pip.conf
 COPY Rprofile /root/.Rprofile
+COPY sources.list /etc/apt/sources.list

integration_tests/sources.list

@@ -0,0 +1 @@
+deb http://localhost:8080/repository/apt-proxy bookworm main

nexus_allowlist/__about__.py

@@ -1 +1 @@
-__version__ = "v0.11.0"
+__version__ = "v0.12.0"

nexus_allowlist/actions.py

@@ -4,6 +4,12 @@
 from pathlib import Path
 
 from nexus_allowlist.nexus import NexusAPI, RepositoryType
+from nexus_allowlist.settings import (
+    APT_DISTRO,
+    APT_REMOTE_URL,
+    CRAN_REMOTE_URL,
+    PYPI_REMOTE_URL,
+)
 
 
 @dataclass
@@ -17,12 +23,17 @@ class Repository:
     "pypi_proxy": Repository(
         repo_type=RepositoryType.PYPI,
         name="pypi-proxy",
-        remote_url="https://pypi.org/",
+        remote_url=PYPI_REMOTE_URL,
     ),
     "cran_proxy": Repository(
         repo_type=RepositoryType.CRAN,
         name="cran-proxy",
-        remote_url="https://cran.r-project.org/",
+        remote_url=CRAN_REMOTE_URL,
+    ),
+    "apt_proxy": Repository(
+        repo_type=RepositoryType.APT,
+        name="apt-proxy",
+        remote_url=APT_REMOTE_URL,
     ),
 }
 
@@ -37,36 +48,44 @@ def check_package_files(args: argparse.Namespace) -> None:
     raise:
         Exception: if any declared allowlist file does not exist
     """
-    for package_file in [args.pypi_package_file, args.cran_package_file]:
+    for package_file in [
+        args.pypi_package_file,
+        args.cran_package_file,
+        args.apt_package_file,
+    ]:
         if package_file and not package_file.is_file():
             msg = f"Package allowlist file {package_file} does not exist"
             raise Exception(msg)
 
 
 def get_allowlists(
-    pypi_package_file: Path, cran_package_file: Path
-) -> tuple[list[str], list[str]]:
+    pypi_package_file: Path, cran_package_file: Path, apt_package_file: Path
+) -> tuple[list[str], list[str], list[str]]:
     """
-    Create allowlists for PyPI and CRAN packages
+    Create allowlists for PyPI, CRAN and APT packages
 
     Args:
         pypi_package_file: Path to the PyPI allowlist file or None
         cran_package_file: Path to the CRAN allowlist file or None
 
     Returns:
-        A tuple of the PyPI and CRAN allowlists (in that order). The lists are
+        A tuple of the PyPI, CRAN and APT allowlists (in that order). The lists are
         [] if the corresponding package file argument was None
     """
     pypi_allowlist = []
     cran_allowlist = []
+    apt_allowlist = []
 
     if pypi_package_file:
         pypi_allowlist = get_allowlist(pypi_package_file, repo_type=RepositoryType.PYPI)
 
     if cran_package_file:
         cran_allowlist = get_allowlist(cran_package_file, repo_type=RepositoryType.CRAN)
 
-    return (pypi_allowlist, cran_allowlist)
+    if apt_package_file:
+        apt_allowlist = get_allowlist(apt_package_file, repo_type=RepositoryType.APT)
+
+    return (pypi_allowlist, cran_allowlist, apt_allowlist)
 
 
 def get_allowlist(allowlist_path: Path, repo_type: RepositoryType) -> list[str]:
@@ -83,9 +102,9 @@ def get_allowlist(allowlist_path: Path, repo_type: RepositoryType) -> list[str]:
     allowlist = []
     with open(allowlist_path) as allowlist_file:
         # Sanitise package names
-        # - convert to lower case if the package is on PyPI. Leave alone on CRAN to
-        #   prevent issues with case-sensitivity - for PyPI replace strings of '.', '_'
-        #   or '-' with '-'
+        # - convert to lower case if the package is on PyPI or APT. Leave alone on CRAN
+        #   to prevent issues with case-sensitivity - for PyPI replace strings of '.',
+        #   '_' or '-' with '-'
         #   https://packaging.python.org/en/latest/guides/distributing-packages-using-setuptools/#name
         # - remove any blank entries, which act as a wildcard that would allow any
         #   package
@@ -94,6 +113,8 @@ def get_allowlist(allowlist_path: Path, repo_type: RepositoryType) -> list[str]:
             match repo_type:
                 case RepositoryType.CRAN:
                     package_name_parsed = package_name.strip()
+                case RepositoryType.APT:
+                    package_name_parsed = package_name.lower().strip()
                 case RepositoryType.PYPI:
                     package_name_parsed = pypi_replace_characters.sub(
                         "-", package_name.lower().strip()
@@ -104,7 +125,7 @@ def get_allowlist(allowlist_path: Path, repo_type: RepositoryType) -> list[str]:
 
 def recreate_repositories(nexus_api: NexusAPI) -> None:
     """
-    Create PyPI and CRAN proxy repositories in an idempotent manner
+    Create PyPI, CRAN and APT proxy repositories in an idempotent manner
 
     Args:
         nexus_api: NexusAPI object
@@ -125,6 +146,7 @@ def recreate_privileges(
     nexus_api: NexusAPI,
     pypi_allowlist: list[str],
     cran_allowlist: list[str],
+    apt_allowlist: list[str],
 ) -> list[str]:
     """
     Create content selectors and content selector privileges based on the
@@ -134,6 +156,7 @@ def recreate_privileges(
         nexus_api: NexusAPI object
         pypi_allowlist: List of allowed PyPI packages
         cran_allowlist: List of allowed CRAN packages
+        apt_allowlist: List of allowed APT packages
 
     Returns:
         List of the names of all content selector privileges
@@ -148,6 +171,7 @@ def recreate_privileges(
 
     pypi_privilege_names = []
     cran_privilege_names = []
+    apt_privilege_names = []
 
     # Content selector and privilege for PyPI 'simple' path, used to search for
     # packages
@@ -185,6 +209,44 @@ def recreate_privileges(
     )
     cran_privilege_names.append(privilege_name)
 
+    # Content selector and privilege for APT 'Packages.gz' file which contains an
+    # metadata for all archived packages
+    privilege_name = create_content_selector_and_privilege(
+        nexus_api,
+        name="apt-packages",
+        description="Allow access to 'Packages.gz' file in APT repository",
+        expression=f'format == "apt" and path=~"^/dists/{APT_DISTRO}/.*/Packages.gz"',
+        repo_type=_NEXUS_REPOSITORIES["apt_proxy"].repo_type,
+        repo=_NEXUS_REPOSITORIES["apt_proxy"].name,
+    )
+    apt_privilege_names.append(privilege_name)
+
+    # Content selector and privilege for APT 'InRelease' file which contains an
+    # metadata about the APT distribution
+    privilege_name = create_content_selector_and_privilege(
+        nexus_api,
+        name="inrelease",
+        description="Allow access to 'InRelease' file in APT repository",
+        expression=f'format == "apt" and path=="/dists/{APT_DISTRO}/InRelease"',
+        repo_type=_NEXUS_REPOSITORIES["apt_proxy"].repo_type,
+        repo=_NEXUS_REPOSITORIES["apt_proxy"].name,
+    )
+    apt_privilege_names.append(privilege_name)
+
+    # Content selector and privilege for APT 'Translation-*' files which contains an
+    # metadata about the APT distribution
+    privilege_name = create_content_selector_and_privilege(
+        nexus_api,
+        name="apt-translation",
+        description="Allow access to 'Translation-*' file in APT repository",
+        expression=(
+            f'format == "apt" and path=~"^/dists/{APT_DISTRO}/.*/Translation-.*"'
+        ),
+        repo_type=_NEXUS_REPOSITORIES["apt_proxy"].repo_type,
+        repo=_NEXUS_REPOSITORIES["apt_proxy"].name,
+    )
+    apt_privilege_names.append(privilege_name)
+
     # Create content selectors and privileges for packages according to the
     # package setting
     if packages == "all":
@@ -209,6 +271,17 @@ def recreate_privileges(
             repo=_NEXUS_REPOSITORIES["cran_proxy"].name,
         )
         cran_privilege_names.append(privilege_name)
+
+        # Allow all APT packages
+        privilege_name = create_content_selector_and_privilege(
+            nexus_api,
+            name="apt-all",
+            description="Allow access to all APT packages",
+            expression='format == "apt" and path=^"/pool/"',
+            repo_type=_NEXUS_REPOSITORIES["apt_proxy"].repo_type,
+            repo=_NEXUS_REPOSITORIES["apt_proxy"].name,
+        )
+        apt_privilege_names.append(privilege_name)
     elif packages == "selected":
         # Allow selected PyPI packages
         for package in pypi_allowlist:
@@ -238,7 +311,19 @@ def recreate_privileges(
             )
             cran_privilege_names.append(privilege_name)
 
-    return pypi_privilege_names + cran_privilege_names
+        # Allow selected APT packages
+        for package in apt_allowlist:
+            privilege_name = create_content_selector_and_privilege(
+                nexus_api,
+                name=f"apt-{package}",
+                description=f"Allow access to {packages} APT package",
+                expression=f'format == "apt" and path=~"^/pool/.*/{package}.*"',
+                repo_type=_NEXUS_REPOSITORIES["apt_proxy"].repo_type,
+                repo=_NEXUS_REPOSITORIES["apt_proxy"].name,
+            )
+            apt_privilege_names.append(privilege_name)
+
+    return pypi_privilege_names + cran_privilege_names + apt_privilege_names
 
 
 def create_content_selector_and_privilege(

nexus_allowlist/cli.py

@@ -76,6 +76,11 @@ def main() -> None:
             "Path of the file of allowed CRAN packages, ignored when PACKAGES is all"
         ),
     )
+    packages_parser.add_argument(
+        "--apt-package-file",
+        type=Path,
+        help="Path of the file of allowed APT packages, ignored when PACKAGES is all",
+    )
 
     subparsers = parser.add_subparsers(title="subcommands", required=True)
 
@@ -168,7 +173,7 @@ def initial_configuration(args: argparse.Namespace) -> None:
 
     This includes:
         - Deleting all respositories
-        - Creating CRAN and PyPI proxies
+        - Creating CRAN, APT and PyPI proxies
         - Deleting all content selectors and content selector privileges
         - Deleting all non-default roles
         - Creating a role
@@ -234,14 +239,14 @@ def update_allow_lists(args: argparse.Namespace) -> None:
     )
 
     # Parse allowlists
-    pypi_allowlist, cran_allowlist = actions.get_allowlists(
-        args.pypi_package_file, args.cran_package_file
+    pypi_allowlist, cran_allowlist, apt_allowlist = actions.get_allowlists(
+        args.pypi_package_file, args.cran_package_file, args.apt_package_file
     )
 
     # Recreate all content selectors and associated privileges according to the
     # allowlists
     privileges = actions.recreate_privileges(
-        args.packages, nexus_api, pypi_allowlist, cran_allowlist
+        args.packages, nexus_api, pypi_allowlist, cran_allowlist, apt_allowlist
     )
 
     # Grant privileges to the nexus allowlist role

nexus_allowlist/nexus.py

@@ -4,6 +4,8 @@
 
 import requests
 
+from nexus_allowlist.settings import APT_DISTRO
+
 _REQUEST_TIMEOUT = 10
 
 
@@ -21,6 +23,7 @@ class ResponseCode(Enum):
 class RepositoryType(Enum):
     PYPI = "pypi"
     CRAN = "r"
+    APT = "apt"
 
 
 class NexusAPI:
@@ -96,7 +99,7 @@ def create_proxy_repository(
         self, repo_type: RepositoryType, name: str, remote_url: str
     ) -> None:
         """
-        Create a proxy repository. Currently supports PyPI and R formats
+        Create a proxy repository. Currently supports PyPI, R and APT formats
 
         Args:
             repo_type: Type of repository
@@ -123,6 +126,8 @@ def create_proxy_repository(
         }
         payload["name"] = name
         payload["proxy"]["remoteUrl"] = remote_url
+        if repo_type == RepositoryType.APT:
+            payload["apt"] = {"distribution": APT_DISTRO, "flat": False}
 
         logging.info(f"Creating {repo_type.value} repository: {name}")
         response = requests.post(