feat: mount the managed instances apt configuration

craft_parts/executor/executor.py

@@ -64,6 +64,7 @@ def __init__(
         ignore_patterns: list[str] | None = None,
         base_layer_dir: Path | None = None,
         base_layer_hash: LayerHash | None = None,
+        use_host_sources: bool = False,
     ) -> None:
         self._part_list = sort_parts(part_list)
         self._project_info = project_info
@@ -73,11 +74,13 @@ def __init__(
         self._base_layer_hash = base_layer_hash
         self._handler: dict[str, PartHandler] = {}
         self._ignore_patterns = ignore_patterns
+        self._use_host_sources = use_host_sources
 
         self._overlay_manager = OverlayManager(
             project_info=self._project_info,
             part_list=self._part_list,
             base_layer_dir=base_layer_dir,
+            use_host_sources=use_host_sources,
         )
 
     def prologue(self) -> None:

craft_parts/lifecycle_manager.py

@@ -108,6 +108,7 @@ def __init__(  # noqa: PLR0913
         project_vars_part_name: str | None = None,
         project_vars: dict[str, str] | None = None,
         partitions: list[str] | None = None,
+        use_host_sources: bool = False,
         **custom_args: Any,  # custom passthrough args
     ) -> None:
         # pylint: disable=too-many-locals
@@ -193,6 +194,7 @@ def __init__(  # noqa: PLR0913
             track_stage_packages=track_stage_packages,
             base_layer_dir=base_layer_dir,
             base_layer_hash=layer_hash,
+            use_host_sources=use_host_sources,
         )
         self._project_info = project_info
         # pylint: enable=too-many-locals

craft_parts/overlays/chroot.py

@@ -20,10 +20,11 @@
 import multiprocessing
 import os
 import sys
-from collections.abc import Callable
+from collections.abc import Callable, Mapping
 from multiprocessing.connection import Connection
 from pathlib import Path
-from typing import Any, NamedTuple
+from shutil import copytree, rmtree
+from typing import Any
 
 from craft_parts.utils import os_utils
 
@@ -33,7 +34,11 @@
 
 
 def chroot(
-    path: Path, target: Callable, *args: Any, **kwargs: Any
+    path: Path,
+    target: Callable,
+    use_host_sources: bool = False,  # noqa: FBT001, FBT002
+    args: tuple[Any] = (),  # type: ignore  # noqa: PGH003
+    kwargs: Mapping[str, Any] = {},
 ) -> Any:  # noqa: ANN401
     """Execute a callable in a chroot environment.
 
@@ -50,14 +55,14 @@ def chroot(
         target=_runner, args=(Path(path), child_conn, target, args, kwargs)
     )
     logger.debug("[pid=%d] set up chroot", os.getpid())
-    _setup_chroot(path)
+    _setup_chroot(path, use_host_sources)
     try:
         child.start()
         res, err = parent_conn.recv()
         child.join()
     finally:
         logger.debug("[pid=%d] clean up chroot", os.getpid())
-        _cleanup_chroot(path)
+        _cleanup_chroot(path, use_host_sources)
 
     if isinstance(err, str):
         raise errors.OverlayChrootExecutionError(err)
@@ -86,87 +91,200 @@ def _runner(
     conn.send((res, None))
 
 
-def _setup_chroot(path: Path) -> None:
+def _compare_os_release(host: os_utils.OsRelease, chroot: os_utils.OsRelease):
+    """Compare OsRelease objects from host and chroot for compatibility. See _host_compatible_chroot."""
+    if (host_val := host.id()) != (chroot_val := chroot.id()):
+        errors.IncompatibleChrootError("id", host_val, chroot_val)
+
+    if (host_val := host.version_id()) != (chroot_val := chroot.version_id()):
+        errors.IncompatibleChrootError("version_id", host_val, chroot_val)
+
+
+def _host_compatible_chroot(path: Path) -> bool:
+    """Raise exception if host and chroot are not the same distrobution and release"""
+    # Note: /etc/os-release is symlinked to /usr/lib/os-release
+    # This could cause an issue if /etc/os-release is removed at any point.
+    host_os_release = os_utils.OsRelease()
+    chroot_os_release = os_utils.OsRelease(
+        os_release_file=str(path / "/etc/os-release")
+    )
+    _compare_os_release(host_os_release, chroot_os_release)
+
+
+def _setup_chroot(path: Path, use_host_sources: bool) -> None:
     """Prepare the chroot environment before executing the target function."""
     logger.debug("setup chroot: %r", path)
     if sys.platform == "linux":
-        _setup_chroot_linux(path)
+        # base configuration
+        _setup_chroot_mounts(path, _linux_mounts)
 
+        if use_host_sources:
+            _host_compatible_chroot(path)
 
-def _cleanup_chroot(path: Path) -> None:
+            _setup_chroot_mounts(path, _ubuntu_apt_mounts)
+
+    logger.debug("chroot setup complete")
+
+
+def _cleanup_chroot(path: Path, use_host_sources: bool) -> None:
     """Clean the chroot environment after executing the target function."""
     logger.debug("cleanup chroot: %r", path)
     if sys.platform == "linux":
-        _cleanup_chroot_linux(path)
+        _cleanup_chroot_mounts(path, _linux_mounts)
+
+        if use_host_sources:
+            # Note: no need to check if host is compatible since
+            # we already called _host_compatible_chroot in _setup_chroot
+            _cleanup_chroot_mounts(path, _ubuntu_apt_mounts)
+
+    logger.debug("chroot cleanup complete")
+
+
+class _Mount:
+    def __init__(
+        self, src: str | Path, mountpoint: str | Path, *args, fstype: str | None = None
+    ) -> None:
+        """Mount entry for chroot setup."""
+
+        self.src = Path(src)
+        self.mountpoint = Path(mountpoint)
+        self.args = list(args)
+
+        if fstype is not None:
+            self.args.append(f"-t{fstype}")
+
+    def _mount(self, src: Path, mountpoint: Path, *args) -> None:
+        mountpoint_str = str(mountpoint)
+        src_str = str(src)
+
+        # Only mount if mountpoint exists.
+        pid = os.getpid()
+        if mountpoint.exists():
+            logger.debug("[pid=%d] mount %r on chroot", pid, mountpoint_str)
+            os_utils.mount(src_str, mountpoint_str, *args)
+        else:
+            logger.error("[pid=%d] mountpoint %r does not exist", pid, mountpoint_str)
+
+    @staticmethod
+    def get_abs_path(path: Path, chroot_path: Path):
+        return path / str(chroot_path).lstrip("/")
+
+    def mount_to(self, path: Path, *args) -> None:
+        abs_mountpoint = self.get_abs_path(path, self.mountpoint)
+
+        self._mount(self.src, abs_mountpoint, *self.args, *args)
+
+    def _umount(self, mountpoint: Path, *args) -> None:
+        mountpoint_str = str(mountpoint)
+
+        pid = os.getpid()
+        if mountpoint.exists():
+            logger.debug("[pid=%d] umount: %r", pid, mountpoint_str)
+            os_utils.umount(mountpoint_str, *args)
+        else:
+            logger.warning("[pid=%d] umount: %r not found!", pid, mountpoint_str)
 
+    def unmount_from(self, path: Path, *args) -> None:
+        abs_mountpoint = self.get_abs_path(path, self.mountpoint)
+        self._umount(abs_mountpoint, *args)
 
-class _Mount(NamedTuple):
-    """Mount entry for chroot setup."""
 
-    fstype: str | None
-    src: str
-    mountpoint: str
-    options: list[str] | None
+class _BindMount(_Mount):
+    bind_type = "bind"
+
+    def __init__(self, src: str | Path, mountpoint: str | Path, *args) -> None:
+        super().__init__(src, mountpoint, f"--{self.bind_type}", *args)
+
+    def _mount(self, src: Path, mountpoint: Path, *args) -> None:
+        if src.is_dir():
+            # remove existing content of dir
+            if mountpoint.exists():
+                rmtree(mountpoint)
+
+            # prep mount point
+            mountpoint.mkdir(parents=True, exist_ok=True)
+
+        elif src.is_file():
+            # remove existing file
+            if mountpoint.exists():
+                mountpoint.unlink()
+            else:
+                mountpoint.parent.mkdir(parents=True, exist_ok=True)
+
+            # prep mount point
+            mountpoint.touch()
+        else:
+            raise FileNotFoundError(f"Path not found: {src}")
+
+        super()._mount(src, mountpoint, *args)
+
+
+class _RBindMount(_BindMount):
+    bind_type = "rbind"
+
+    def _umount(self, mountpoint: Path, *args) -> None:
+        super()._umount(mountpoint, "--recursive", "--lazy", *args)
+
+
+class _TempFSClone(_Mount):
+    def __init__(self, src: str, mountpoint: str, *args) -> None:
+        super().__init__(src, mountpoint, *args, fstype="tmpfs")
+
+    def _mount(self, src: Path, mountpoint: Path, *args) -> None:
+        if src.is_dir():
+            # remove existing content of dir
+            if mountpoint.exists():
+                rmtree(mountpoint)
+
+            # prep mount point
+            mountpoint.mkdir(parents=True, exist_ok=True)
+
+        elif src.is_file():
+            raise NotADirectoryError(f"Path is a directory: {src}")
+        else:
+            raise FileNotFoundError(f"Path not found: {src}")
+
+        super()._mount(src, mountpoint, *args)
+
+        copytree(src, mountpoint, dirs_exist_ok=True)
 
 
 # Essential filesystems to mount in order to have basic utilities and
 # name resolution working inside the chroot environment.
+#
+# Some images (such as cloudimgs) symlink ``/etc/resolv.conf`` to
+# ``/run/systemd/resolve/stub-resolv.conf``. We want resolv.conf to be
+# a regular file to bind-mount the host resolver configuration on.
+#
+# There's no need to restore the file to its original condition because
+# this operation happens on a temporary filesystem layer.
 _linux_mounts: list[_Mount] = [
-    _Mount(None, "/etc/resolv.conf", "/etc/resolv.conf", ["--bind"]),
-    _Mount("proc", "proc", "/proc", None),
-    _Mount("sysfs", "sysfs", "/sys", None),
+    _BindMount("/etc/resolv.conf", "/etc/resolv.conf"),
+    _Mount("proc", "/proc", fstype="proc"),
+    _Mount("sysfs", "/sys", fstype="sysfs"),
     # Device nodes require MS_REC to be bind mounted inside a container.
-    _Mount(None, "/dev", "/dev", ["--rbind", "--make-rprivate"]),
+    _RBindMount("/dev", "/dev", "--make-rprivate"),
 ]
 
+# Mounts required to import host's Ubuntu Pro apt configuration to chroot
+# TODO: parameterize this per linux distribution / package manager
+_ubuntu_apt_mounts = [
+    _TempFSClone("/etc/apt", "/etc/apt"),
+    _BindMount("/usr/share/ca-certificates/", "/usr/share/ca-certificates/"),
+    _BindMount("/etc/ssl/certs/", "/etc/ssl/certs/"),
+    _BindMount("/etc/ca-certificates.conf", "/etc/ca-certificates.conf"),
+]
 
-def _setup_chroot_linux(path: Path) -> None:
-    """Linux-specific chroot environment preparation."""
-    # Some images (such as cloudimgs) symlink ``/etc/resolv.conf`` to
-    # ``/run/systemd/resolve/stub-resolv.conf``. We want resolv.conf to be
-    # a regular file to bind-mount the host resolver configuration on.
-    #
-    # There's no need to restore the file to its original condition because
-    # this operation happens on a temporary filesystem layer.
-    resolv_conf = path / "etc/resolv.conf"
-    if resolv_conf.is_symlink():
-        resolv_conf.unlink()
-        resolv_conf.touch()
-    elif not resolv_conf.exists() and resolv_conf.parent.is_dir():
-        resolv_conf.touch()
-
-    pid = os.getpid()
-    for entry in _linux_mounts:
-        args = []
-        if entry.options:
-            args.extend(entry.options)
-        if entry.fstype:
-            args.append(f"-t{entry.fstype}")
-
-        mountpoint = path / entry.mountpoint.lstrip("/")
 
-        # Only mount if mountpoint exists.
-        if mountpoint.exists():
-            logger.debug("[pid=%d] mount %r on chroot", pid, str(mountpoint))
-            os_utils.mount(entry.src, str(mountpoint), *args)
-        else:
-            logger.debug("[pid=%d] mountpoint %r does not exist", pid, str(mountpoint))
+def _setup_chroot_mounts(path: Path, mounts: list[_Mount]) -> None:
+    """Linux-specific chroot environment preparation."""
 
-    logger.debug("chroot setup complete")
+    for entry in mounts:
+        entry.mount_to(path)
 
 
-def _cleanup_chroot_linux(path: Path) -> None:
+def _cleanup_chroot_mounts(path: Path, mounts: list[_Mount]) -> None:
     """Linux-specific chroot environment cleanup."""
-    pid = os.getpid()
-    for entry in reversed(_linux_mounts):
-        mountpoint = path / entry.mountpoint.lstrip("/")
 
-        if mountpoint.exists():
-            logger.debug("[pid=%d] umount: %r", pid, str(mountpoint))
-            if entry.options and "--rbind" in entry.options:
-                # Mount points under /dev may be in use and make the bind mount
-                # unmountable. This may happen in destructive mode depending on
-                # the host environment, so use MNT_DETACH to defer unmounting.
-                os_utils.umount(str(mountpoint), "--recursive", "--lazy")
-            else:
-                os_utils.umount(str(mountpoint))
+    for entry in reversed(mounts):
+        entry.unmount_from(path)

craft_parts/overlays/errors.py

@@ -61,3 +61,18 @@ def __init__(self, message: str) -> None:
         brief = f"Overlay environment execution error: {message}"
 
         super().__init__(brief=brief)
+
+
+class IncompatibleChrootError(OverlayError):
+    """Failed to use host package sources because chroot is incompatible.
+
+    :param key: os-release key which was tested.
+    :param host: value from host os-release which was tested.
+    :param chroot: value from chroot os-release which was tested.
+    """
+
+    def __init__(self, key: str, host: str, chroot: str) -> None:
+        self.message = f"key {key} in os-release expected to be {host} found {chroot}"
+        brief = f"Unable to use host sources: {self.message}"
+
+        super().__init__(brief=brief)