diff --git a/.github/workflows/tests.yaml b/.github/workflows/tests.yaml index 2f2e208f8..33070fb58 100644 --- a/.github/workflows/tests.yaml +++ b/.github/workflows/tests.yaml @@ -66,12 +66,33 @@ jobs: sudo apt install -y python3-pip python3-setuptools python3-wheel python3-venv libapt-pkg-dev export $(cat /etc/os-release | grep VERSION_CODENAME) pip install -U -r "requirements-${VERSION_CODENAME}.txt" + - name: Install external dependencies with homebrew + # This is only necessary for Linux until skopeo >= 1.11 is in repos. + # Once we're running on Noble, we can get skopeo from apt. + if: ${{ runner.os == 'Linux' || runner.os == 'macOS' }} + run: | + if [[ $(uname --kernel-name) == "Linux" ]]; then + eval "$(/home/linuxbrew/.linuxbrew/bin/brew shellenv)" + fi + brew install skopeo - name: Configure environment run: | python -m pip install tox tox run --colored yes -m tests --notest - name: Run tests + shell: bash run: | + if [[ $(uname --kernel-name) == "Linux" ]]; then + # Ensure the version of skopeo comes from homebrew + # This is only necessary until we move to noble. + eval "$(/home/linuxbrew/.linuxbrew/bin/brew shellenv)" + # Allow skopeo to access the contents of /run/containers + sudo chmod 777 /run/containers + # Add an xdg runtime dir for skopeo to look into for an auth.json file + sudo mkdir -p /run/user/$(id -u) + sudo chown $USER /run/user/$(id -u) + export XDG_RUNTIME_DIR=/run/user/$(id -u) + fi tox run --skip-pkg-install --no-list-dependencies --result-json results/tox-${{ matrix.platform }}.json --colored yes -m tests snap-build: diff --git a/charmcraft/application/commands/store.py b/charmcraft/application/commands/store.py index a0df916d7..23189785f 100644 --- a/charmcraft/application/commands/store.py +++ b/charmcraft/application/commands/store.py @@ -20,6 +20,7 @@ import dataclasses import os import pathlib +import re import shutil import string import tempfile @@ -45,7 +46,7 @@ from charmcraft import const, env, errors, parts, utils from charmcraft.application.commands.base import CharmcraftCommand from charmcraft.models import project -from charmcraft.store import ImageHandler, LocalDockerdInterface, OCIRegistry, Store +from charmcraft.store import Store from charmcraft.store.models import Entity from charmcraft.utils import cli @@ -1923,7 +1924,7 @@ def fill_parser(self, parser): help="The architectures valid for this file resource. If none are provided, the resource is uploaded without architecture information.", ) - def run(self, parsed_args): + def run(self, parsed_args: argparse.Namespace) -> int: """Run the command.""" store = Store(env.get_store_config()) @@ -1938,87 +1939,97 @@ def run(self, parsed_args): architectures = ["all"] if parsed_args.filepath: - resource_filepath = parsed_args.filepath - resource_filepath_is_temp = False - resource_type = ResourceType.file - emit.progress(f"Uploading resource directly from file {str(resource_filepath)!r}.") + emit.progress(f"Uploading resource directly from file {str(parsed_args.filepath)!r}.") bases = [{"name": "all", "channel": "all", "architectures": architectures}] + result = store.upload_resource( + parsed_args.charm_name, + parsed_args.resource_name, + ResourceType.file, + parsed_args.filepath, + bases=bases, + ) elif parsed_args.image: + emit.progress("Getting image") + emit.debug("Trying to get image from Docker") + image_service = self._services.image + # Check Docker first for backwards compatibility - prefer to get from + # Docker than from a local path if Docker contains the image. + if digest := image_service.get_maybe_id_from_docker(parsed_args.image): + emit.debug("Image is available via the local Docker daemon.") + source_path = f"docker-daemon:{digest}" + elif (image_path := pathlib.Path(parsed_args.image)).exists(): + emit.debug("Image is a path.") + if image_path.is_file(): + emit.debug("Image is a rock or other OCI archive.") + source_path = f"oci-archive:{image_path.as_posix()}" + elif image_path.is_dir(): + emit.debug("Image is an OCI directory.") + source_path = f"oci:{image_path.as_posix()}" + else: + raise CraftError( + f"Not a valid file or directory: {image_path.as_posix()!r}", + resolution="Pass an OCI archive file such as a rock.", + ) + elif re.match("^[a-z-]:", parsed_args.image): + emit.debug("Presuming an OCI path that skopeo understands.") + source_path = parsed_args.image + else: + raise CraftError( + "Unknown OCI image reference.", + details="Passed image reference is not a Docker image ID, image digest, existing file or container transport string.", + resolution="Pass a valid container transport string.", + ) + emit.debug(f"Using source path {source_path!r}") + + emit.progress("Inspecting source image") + image_metadata = image_service.inspect(source_path) + credentials = store.get_oci_registry_credentials( parsed_args.charm_name, parsed_args.resource_name ) - # convert the standard OCI registry image name (which is something like - # 'registry.jujucharms.com/charm/45kk8smbiyn2e/redis-image') to the image - # name that we use internally (just remove the initial "server host" part) - image_name = credentials.image_name.split("/", 1)[1] - emit.progress(f"Uploading resource from image {image_name} @ {parsed_args.image}.") - - # build the image handler and dockerd interface - registry = OCIRegistry( - env.get_store_config().registry_url, - image_name, - username=credentials.username, - password=credentials.password, - ) - ih = ImageHandler(registry) - dockerd = LocalDockerdInterface() - - server_image_digest = None - if ":" in parsed_args.image: - # the user provided a digest; check if the specific image is - # already in Canonical's registry - already_uploaded = ih.check_in_registry(parsed_args.image) - - if already_uploaded: - emit.progress("Using OCI image from Canonical's registry.", permanent=True) - server_image_digest = parsed_args.image - else: - emit.progress("Remote image not found, getting its info from local registry.") - image_info = dockerd.get_image_info_from_digest(parsed_args.image) - + if const.STORE_REGISTRY_ENV_VAR in os.environ: + # If the user has specified a registry to use, replace what the store + # gives with that registry. + registry_url = os.environ[const.STORE_REGISTRY_ENV_VAR][7:] + image_name = credentials.image_name.split("/", 1)[1] + dest_path = f"docker://{registry_url}/{image_name}" else: - # the user provided an id, can't search remotely, just get its info locally - emit.progress("Getting image info from local registry.") - image_info = dockerd.get_image_info_from_id(parsed_args.image) - - if server_image_digest is None: - if image_info is None: - raise CraftError("Image not found locally.") - - # upload it from local registry - emit.progress("Uploading from local registry.", permanent=True) - server_image_digest = ih.upload_from_local(image_info) - emit.progress( - f"Image uploaded, new remote digest: {server_image_digest}.", permanent=True + dest_path = f"docker://{credentials.image_name}" + + with emit.open_stream("Uploading") as stream: + image_service.copy( + source_path, + dest_path, + stream, + dest_username=credentials.username, + dest_password=credentials.password, ) + image_arch = [ + utils.ARCH_TRANSLATIONS.get(arch, arch) for arch in image_metadata.architectures + ] + bases = [{"name": "all", "channel": "all", "architectures": image_arch}] + # all is green, get the blob to upload to Charmhub content = store.get_oci_image_blob( - parsed_args.charm_name, parsed_args.resource_name, server_image_digest + parsed_args.charm_name, parsed_args.resource_name, image_metadata.digest ) - tfd, tname = tempfile.mkstemp(prefix="image-resource", suffix=".json") - with open(tfd, "w", encoding="utf-8") as fh: # reuse the file descriptor and close it - fh.write(content) - resource_filepath = pathlib.Path(tname) - resource_filepath_is_temp = True - resource_type = ResourceType.oci_image - - image_arch = image_info.get("Architecture", "all") - image_arch = utils.ARCH_TRANSLATIONS.get(image_arch, image_arch) - bases = [{"name": "all", "channel": "all", "architectures": [image_arch]}] - - result = store.upload_resource( - parsed_args.charm_name, - parsed_args.resource_name, - resource_type, - resource_filepath, - bases=bases, - ) - - # clean the filepath if needed - if resource_filepath_is_temp: - resource_filepath.unlink() + with tempfile.NamedTemporaryFile( + mode="w+", prefix="image-resource", suffix=".json" + ) as resource_file: + resource_file.write(content) + resource_file.flush() + + result = store.upload_resource( + parsed_args.charm_name, + parsed_args.resource_name, + ResourceType.oci_image, + pathlib.Path(resource_file.name), + bases=bases, + ) + else: + raise CraftError("Either a file path or an image descriptor must be passed.") if result.ok: if parsed_args.format: diff --git a/charmcraft/application/main.py b/charmcraft/application/main.py index 1768cfcbf..9c85e54bd 100644 --- a/charmcraft/application/main.py +++ b/charmcraft/application/main.py @@ -49,7 +49,7 @@ def __init__( app: AppMetadata, services: CharmcraftServiceFactory, ) -> None: - super().__init__(app=app, services=services) + super().__init__(app=app, services=services, extra_loggers={"charmcraft"}) self._global_args: dict[str, Any] = {} self._dispatcher: craft_cli.Dispatcher | None = None self._cli_loggers |= {"charmcraft"} diff --git a/charmcraft/errors.py b/charmcraft/errors.py index a12af5c04..edf7a6361 100644 --- a/charmcraft/errors.py +++ b/charmcraft/errors.py @@ -16,10 +16,14 @@ """Charmcraft error classes.""" import io import pathlib +import shlex +import subprocess +import textwrap from collections.abc import Iterable, Mapping -from typing import TYPE_CHECKING +from typing import TYPE_CHECKING, cast from craft_cli import CraftError +from typing_extensions import Self if TYPE_CHECKING: from charmcraft.linters import CheckResult @@ -171,3 +175,22 @@ def __init__(self, extra_dependencies: Iterable[str]): class ExtensionError(CraftError): """Error related to extension handling.""" + + +class SubprocessError(CraftError): + """A craft-cli friendly subprocess error.""" + + @classmethod + def from_subprocess(cls, error: subprocess.CalledProcessError) -> Self: + """Convert a CalledProcessError to a craft-cli error.""" + error_details = f"Full command: {shlex.join(error.cmd)}\nError text:\n" + if isinstance(error.stderr, str): + error_details += textwrap.indent(error.stderr, " ") + else: + stderr = cast(io.TextIOBase, error.stderr) + stderr.seek(io.SEEK_SET) + error_details += textwrap.indent(stderr.read(), " ") + return cls( + f"Error while running {error.cmd[0]} (return code {error.returncode})", + details=error_details, + ) diff --git a/charmcraft/services/__init__.py b/charmcraft/services/__init__.py index 25218adab..6f00f865d 100644 --- a/charmcraft/services/__init__.py +++ b/charmcraft/services/__init__.py @@ -23,6 +23,7 @@ from craft_application import ServiceFactory from .analysis import AnalysisService +from .image import ImageService from .lifecycle import LifecycleService from .package import PackageService from .provider import ProviderService @@ -40,20 +41,23 @@ class CharmcraftServiceFactory(ServiceFactory): AnalysisClass: type[AnalysisService] = AnalysisService StoreClass: type[StoreService] = StoreService RemoteBuildClass: type[RemoteBuildService] = RemoteBuildService + ImageClass: type[ImageService] = ImageService if TYPE_CHECKING: # Cheeky hack that lets static type checkers report the correct types. # Any apps that add their own services should do this too. analysis: AnalysisService = None # type: ignore[assignment] - package: PackageService = None # type: ignore[assignment] + image: ImageService = None # type: ignore[assignment] lifecycle: LifecycleService = None # type: ignore[assignment] - provider: ProviderService = None # type: ignore[assignment] + package: PackageService = None # type: ignore[assignment] project: models.CharmcraftProject = None # type: ignore[assignment] + provider: ProviderService = None # type: ignore[assignment] store: StoreService = None # type: ignore[assignment] __all__ = [ "AnalysisService", + "ImageService", "LifecycleService", "PackageService", "ProviderService", diff --git a/charmcraft/services/image.py b/charmcraft/services/image.py new file mode 100644 index 000000000..9a2306eaf --- /dev/null +++ b/charmcraft/services/image.py @@ -0,0 +1,145 @@ +# Copyright 2024 Canonical Ltd. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# For further info, check https://github.com/canonical/charmcraft +"""Service class for handling OCI images.""" + +import dataclasses +import io +import logging +import subprocess +from collections.abc import Sequence + +import craft_application +import docker # type: ignore[import-untyped] +import docker.errors # type: ignore[import-untyped] +import docker.models.images # type: ignore[import-untyped] +from overrides import override + +from charmcraft import const, errors, utils + +logger = logging.getLogger("charmcraft") + + +@dataclasses.dataclass(frozen=True) +class OCIMetadata: + """Metadata about an OCI image. + + :param path: A skopeo-compatible image path + :param digest: A digest string for the image + :param architectures: A list of architectures for Linux platforms of the image. + """ + + path: str + digest: str + architectures: Sequence[const.CharmArch] + + +class ImageService(craft_application.AppService): + """Service to handle OCI images. + + This service is mostly a craft-application friendly wrapper around skopeo. + Use of this service requires the app to have access to skopeo. + """ + + _skopeo: utils.Skopeo + _docker: docker.DockerClient + + @override + def setup(self) -> None: + """Set up the image service.""" + super().setup() + self._skopeo = utils.Skopeo() + self._docker = docker.from_env() + + def copy( + self, + source_image: str, + destination_image: str, + stdout: io.FileIO, + *, + dest_username: str | None = None, + dest_password: str | None = None, + ): + """Use Skopeo to copy an image. + + :param source_image: A skopeo-accepted source image string. + :param destination_image: A skopeo-accepted destination image string. + :param stdout: A stream to use as skopeo's stdout + + This is designed to be used with craft-cli's emit.open_stream or similar. + For example: + >>> with emit.open_stream("Uploading image") as stream: + ... image_service.copy(source, dest, stdout=stream) + However, it can be used in other ways by passing any file-like object as the stream. + """ + self._skopeo.copy( + source_image=source_image, + destination_image=destination_image, + stdout=stdout, + stderr=subprocess.PIPE, + dest_username=dest_username, + dest_password=dest_password, + preserve_digests=True, + ) + + def get_maybe_id_from_docker(self, name: str) -> str | None: + """Get the ID of an image from Docker. + + :param name: Any string Docker recognises as the image name. + :returns: An image digest or None + + The digest will match the OCI digest spec: + https://github.com/opencontainers/image-spec/blob/main/descriptor.md#digests + """ + try: + image = self._docker.images.get(name) + except docker.errors.ImageNotFound: + return None + return image.id + + def inspect(self, image: str) -> OCIMetadata: + """Inspect an image with Skopeo and return the relevant metadata. + + :param image: A skopeo-friendly image name + :returns: An OCIMetadata object containing metadata about the image. + """ + try: + raw_manifest = self._skopeo.inspect(image, raw=True) + image_info = self._skopeo.inspect(image) + except errors.SubprocessError as exc: + raise errors.CraftError( + "Could not inspect OCI image.", details=f"{exc.message}\n{exc.details}" + ) + try: + image_digest = image_info["Digest"] + except KeyError: + raise errors.CraftError("Could not get digest for image.") + if "manifests" in raw_manifest: # Multi-arch OCI image + architectures: list[const.CharmArch] = [] + for child in raw_manifest["manifests"]: + platform = child.get("platform", {}) + if platform.get("os") != "linux": + continue + architectures.append(const.CharmArch(platform["architecture"])) + if not architectures: + raise errors.CraftError("No architectures found in image for Linux OS.") + else: + architectures = [const.CharmArch(image_info["Architecture"])] + + return OCIMetadata( + path=image, + digest=image_digest, + architectures=architectures, + ) diff --git a/charmcraft/store/store.py b/charmcraft/store/store.py index 0722bd993..2e8517734 100644 --- a/charmcraft/store/store.py +++ b/charmcraft/store/store.py @@ -478,7 +478,9 @@ def list_resource_revisions( return self._client.list_resource_revisions(charm_name, resource_name) @_store_client_wrapper() - def get_oci_registry_credentials(self, charm_name, resource_name): + def get_oci_registry_credentials( + self, charm_name: str, resource_name: str + ) -> RegistryCredentials: """Get credentials to upload a resource to the Canonical's OCI Registry.""" endpoint = f"/v1/charm/{charm_name}/resources/{resource_name}/oci-image/upload-credentials" response = self._client.request_urlpath_json("GET", endpoint) diff --git a/charmcraft/utils/__init__.py b/charmcraft/utils/__init__.py index f9bac33a1..82907482f 100644 --- a/charmcraft/utils/__init__.py +++ b/charmcraft/utils/__init__.py @@ -62,6 +62,7 @@ get_charm_name_from_path, get_templates_environment, ) +from charmcraft.utils.skopeo import Skopeo from charmcraft.utils.store import get_packages from charmcraft.utils.yaml import dump_yaml, load_yaml @@ -106,6 +107,7 @@ "find_charm_sources", "get_charm_name_from_path", "get_templates_environment", + "Skopeo", "get_packages", "dump_yaml", "load_yaml", diff --git a/charmcraft/utils/file.py b/charmcraft/utils/file.py index d4e20bba7..cb4d5be38 100644 --- a/charmcraft/utils/file.py +++ b/charmcraft/utils/file.py @@ -18,8 +18,8 @@ import os import pathlib import zipfile - from _stat import S_IRGRP, S_IROTH, S_IRUSR, S_IXGRP, S_IXOTH, S_IXUSR + from craft_cli import CraftError # handy masks for execution and reading for everybody diff --git a/charmcraft/utils/skopeo.py b/charmcraft/utils/skopeo.py new file mode 100644 index 000000000..3bdaebb06 --- /dev/null +++ b/charmcraft/utils/skopeo.py @@ -0,0 +1,153 @@ +# Copyright 2024 Canonical Ltd. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# For further info, check https://github.com/canonical/charmcraft +"""A wrapper around Skopeo.""" + +import io +import json +import pathlib +import shutil +import subprocess +from collections.abc import Sequence +from typing import Any, cast, overload + +from charmcraft import errors + + +class Skopeo: + """A class for interacting with skopeo.""" + + def __init__( + self, + *, + skopeo_path: str = "", + insecure_policy: bool = False, + arch: str | None = None, + os: str | None = None, + tmpdir: pathlib.Path | None = None, + debug: bool = False, + ) -> None: + if skopeo_path: + self._skopeo = skopeo_path + else: + self._skopeo = cast(str, shutil.which("skopeo")) + if not self._skopeo: + raise RuntimeError("Cannot find a skopeo executable.") + self._insecure_policy = insecure_policy + self.arch = arch + self.os = os + if tmpdir: + tmpdir.mkdir(parents=True, exist_ok=True) + self._tmpdir = tmpdir + self._debug = debug + + self._run_skopeo([self._skopeo, "--version"], capture_output=True, text=True) + + def get_global_command(self) -> list[str]: + """Prepare the global skopeo options.""" + command = [self._skopeo] + if self._insecure_policy: + command.append("--insecure-policy") + if self.arch: + command.extend(["--override-arch", self.arch]) + if self.os: + command.extend(["--override-os", self.os]) + if self._tmpdir: + command.extend(["--tmpdir", str(self._tmpdir)]) + if self._debug: + command.append("--debug") + return command + + def _run_skopeo(self, command: Sequence[str], **kwargs) -> subprocess.CompletedProcess: + """Run skopeo, converting the error message if necessary.""" + try: + return subprocess.run(command, check=True, **kwargs) + except subprocess.CalledProcessError as exc: + raise errors.SubprocessError.from_subprocess(exc) from exc + + def copy( + self, + source_image: str, + destination_image: str, + *, + all_images: bool = False, + preserve_digests: bool = False, + source_username: str | None = None, + source_password: str | None = None, + dest_username: str | None = None, + dest_password: str | None = None, + stdout: io.FileIO | int | None = None, + stderr: io.FileIO | int | None = None, + ) -> subprocess.CompletedProcess: + """Copy an OCI image using Skopeo.""" + command = [ + *self.get_global_command(), + "copy", + ] + if all_images: + command.append("--all") + if preserve_digests: + command.append("--preserve-digests") + if source_username and source_password: + command.extend(["--src-creds", f"{source_username}:{source_password}"]) + elif source_username: + command.extend(["--src-creds", source_username]) + elif source_password: + command.extend(["--src-password", source_password]) + if dest_username and dest_password: + command.extend(["--dest-creds", f"{dest_username}:{dest_password}"]) + elif dest_username: + command.extend(["--dest-creds", dest_username]) + elif dest_password: + command.extend(["--dest-password", dest_password]) + + command.extend([source_image, destination_image]) + + if stdout or stderr: + return self._run_skopeo(command, stdout=stdout, stderr=stderr, text=True) + return self._run_skopeo(command, capture_output=True, text=True) + + @overload + def inspect( + self, image: str, *, format_template: None = None, raw: bool = False, tags: bool = True + ) -> dict[str, Any]: ... + @overload + def inspect( + self, image: str, *, format_template: str, raw: bool = False, tags: bool = True + ) -> str: ... + def inspect( + self, + image: str, + *, + format_template: str | None = None, + raw: bool = False, + tags: bool = True, + ) -> dict[str, Any] | str: + """Inspect an image.""" + command = [*self.get_global_command(), "inspect"] + if format_template is not None: + command.extend(["--format", format_template]) + if raw: + command.append("--raw") + if not tags: + command.append("--no-tags") + + command.append(image) + + result = self._run_skopeo(command, capture_output=True, text=True) + + if format_template is None: + return json.loads(result.stdout) + return result.stdout diff --git a/pyproject.toml b/pyproject.toml index 086ba5b55..8329b6e31 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -9,6 +9,7 @@ dependencies = [ "craft-providers>=1.23.0", "craft-store>=2.4", "distro>=1.3.0", + "docker>=7.0.0", "humanize>=2.6.0", "jsonschema", "jinja2", diff --git a/requirements-dev.txt b/requirements-dev.txt index 194a3df51..9f80eae97 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -13,6 +13,7 @@ craft-store==2.6.2 cryptography==42.0.8 Deprecated==1.2.14 distro==1.9.0 +docker==7.1.0 flake8==7.0.0 freezegun==1.5.1 httplib2==0.22.0 diff --git a/requirements.txt b/requirements.txt index 152c4ee66..e88b770bc 100644 --- a/requirements.txt +++ b/requirements.txt @@ -12,6 +12,7 @@ craft-store==2.6.2 cryptography==42.0.8 Deprecated==1.2.14 distro==1.9.0 +docker==7.1.0 httplib2==0.22.0 humanize==4.9.0 idna==3.7 diff --git a/snap/snapcraft.yaml b/snap/snapcraft.yaml index abcbfcfe7..2fcc542f7 100644 --- a/snap/snapcraft.yaml +++ b/snap/snapcraft.yaml @@ -152,6 +152,32 @@ parts: organize: bin/craftctl: libexec/charmcraft/craftctl + skopeo: # Copied from Rockcraft + plugin: nil + source: https://github.com/containers/skopeo.git + source-tag: v1.15.1 + override-build: | + CGO=1 go build -ldflags -linkmode=external ./cmd/skopeo + mkdir "$CRAFT_PART_INSTALL"/bin + install -m755 skopeo "$CRAFT_PART_INSTALL"/bin/skopeo + stage-packages: + - libgpgme11 + - libassuan0 + - libbtrfs0 + - libdevmapper1.02.1 + build-attributes: + - enable-patchelf + build-snaps: + - go/1.21/stable + build-packages: + - libgpgme-dev + - libassuan-dev + - libbtrfs-dev + - libdevmapper-dev + - pkg-config + organize: + bin/skopeo: libexec/charmcraft/skopeo + spread: plugin: go source: https://github.com/snapcore/spread.git diff --git a/tests/integration/utils/__init__.py b/tests/integration/utils/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/tests/integration/utils/test_skopeo.py b/tests/integration/utils/test_skopeo.py new file mode 100644 index 000000000..281827a5b --- /dev/null +++ b/tests/integration/utils/test_skopeo.py @@ -0,0 +1,87 @@ +# Copyright 2024 Canonical Ltd. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# For further info, check https://github.com/canonical/charmcraft +"""Integration tests for skopeo wrapper.""" + +import contextlib +import os +import pathlib +import platform +import shutil + +import pytest + +from charmcraft import errors +from charmcraft.utils.skopeo import Skopeo + +pytestmark = [ + pytest.mark.skipif( + "CI" not in os.environ and not shutil.which("skopeo"), reason="skopeo not found in PATH" + ), + pytest.mark.xfail( + platform.system().lower() not in ("linux", "darwin"), + reason="Don't necessarily have skopeo on non Linux/mac platforms.", + strict=False, # Allow them to pass anyway. + ), +] + + +@pytest.mark.parametrize( + ("name", "image", "tag"), + [ + ("alpine", "docker://ghcr.io/containerd/alpine", "3.14.0"), + ("debian12", "docker://gcr.io/distroless/base-debian12", "nonroot"), + ("mock-rock", "docker://ghcr.io/canonical/oci-factory/mock-rock", "1.2-22.04_279"), + ("nanoserver", "docker://ghcr.io/containerd/nanoserver", "1809"), + ], +) +def test_inspect_and_download( + monkeypatch, tmp_path: pathlib.Path, name: str, image: str, tag: str +): + (tmp_path / "tmp").mkdir() + skopeo = Skopeo(tmpdir=tmp_path / "tmp") + + raw_data = skopeo.inspect(f"{image}:{tag}", raw=True) + + with contextlib.suppress(errors.SubprocessError): + # These fail if the host platform doesn't match a valid platform for the container. + meta_data = skopeo.inspect(f"{image}:{tag}") + meta_data_by_hash = skopeo.inspect(f"{image}@{meta_data['Digest']}") + assert meta_data == meta_data_by_hash + raw_data_by_hash = skopeo.inspect(f"{image}@{meta_data['Digest']}", raw=True) + assert raw_data == raw_data_by_hash + + for manifest in raw_data["manifests"]: + if "variant" in manifest["platform"]: + # We don't handle variants currently. + continue + os = manifest["platform"]["os"] + arch = manifest["platform"]["architecture"] + digest = manifest["digest"] + by_digest_tar = tmp_path / f"{name}_{os}_{arch}-digest.tar" + by_os_arch_tar = tmp_path / f"{name}_{os}_{arch}.tar" + os_arch_skopeo = Skopeo(arch=arch, os=os) + + os_arch_skopeo.copy(f"{image}:{tag}", f"oci-archive:{by_os_arch_tar}") + skopeo.copy(f"{image}@{digest}", f"oci-archive:{by_digest_tar}") + + assert by_digest_tar.exists() + assert by_os_arch_tar.exists() + + image_info_by_digest = skopeo.inspect(f"{image}@{digest}") + image_info_by_os_arch = os_arch_skopeo.inspect(f"{image}:{tag}") + + for key in ["Architecture", "Os", "Layers", "LayersData", "Env"]: + assert image_info_by_digest[key] == image_info_by_os_arch[key] diff --git a/tests/spread/store/resources/task.yaml b/tests/spread/store/resources/task.yaml index fea44e8a3..6cc8656c7 100644 --- a/tests/spread/store/resources/task.yaml +++ b/tests/spread/store/resources/task.yaml @@ -80,12 +80,31 @@ execute: | new_architectures=$(charmcraft set-resource-architectures $CHARM_DEFAULT_NAME example-file --format=json --revision=$upload_revision_number all | jq -c '.[0].architectures') [[ "$new_architectures" == '["all"]' ]] - # upload and verify the OCI image resource - charmcraft upload-resource $CHARM_DEFAULT_NAME example-image --image=sha256:18a657d0cc1c7d0678a3fbea8b7eb4918bba25968d3e1b0adebfa71caddbc346 + # upload and verify the OCI image resource by digest + # This is a different image hash because it's what Docker contains locally on amd64. + uploaded_revision=$(charmcraft --verbose upload-resource --format=json $CHARM_DEFAULT_NAME example-image --image=sha256:feb5d9fea6a5e9606aa995e879d862b825965ba48de054caab5ef356dc6b3412 | jq -r '.revision') + sleep 3 # Give the store a bit of time to process it last_revision=$(charmcraft resource-revisions $CHARM_DEFAULT_NAME example-image --format=json | jq -r .[0]) last_image_revno=$(echo $last_revision | jq .revision) + test $last_image_revno -ge $uploaded_revision last_revision_created=$(echo $last_revision | jq -r '.["created at"]') + # Upload and verify the OCI image resource by image ID. + uploaded_revision=$(charmcraft upload-resource --format=json $CHARM_DEFAULT_NAME example-image --image=feb5d9fea6a5 | jq -r '.revision') + sleep 3 # Give the store a bit of time to process it + last_revision=$(charmcraft resource-revisions $CHARM_DEFAULT_NAME example-image --format=json | jq -r .[0]) + last_image_revno=$(echo $last_revision | jq .revision) + test $last_image_revno -ge $uploaded_revision + last_revision_created=$(echo $last_revision | jq -r '.["created at"]') + + # Upload and verify a docker image resource file. + docker save hello-world@sha256:18a657d0cc1c7d0678a3fbea8b7eb4918bba25968d3e1b0adebfa71caddbc346 -o hello-world.docker + docker_revision=$(charmcraft upload-resource --format=json $CHARM_DEFAULT_NAME example-image --image=docker-archive:hello-world.docker | jq -r '.revision') + /snap/charmcraft/current/libexec/charmcraft/skopeo copy docker-archive:hello-world.docker oci-archive:hello-world.tar + oci_revision=$(charmcraft upload-resource --format=json $CHARM_DEFAULT_NAME example-image --image=oci-archive:hello-world.tar | jq -r '.revision') + test $docker_revision -eq $uploaded_revision + test $oci_revision -eq $uploaded_revision + # release and check full status charmcraft release $CHARM_DEFAULT_NAME -r $last_charm_revno -c edge --resource=example-file:$last_file_revno --resource=example-image:$last_image_revno edge_release=$(charmcraft status $CHARM_DEFAULT_NAME --format=json | jq -r '.[] | select(.track=="latest") | .mappings[0].releases | .[] | select(.channel=="latest/edge")') diff --git a/tests/unit/commands/test_store.py b/tests/unit/commands/test_store.py index fe69d988a..8e9102e5f 100644 --- a/tests/unit/commands/test_store.py +++ b/tests/unit/commands/test_store.py @@ -164,7 +164,7 @@ def test_fetch_libs_no_charm_libs( Could not find the following libraries on charmhub: - lib: mysql.mysql version: '1' - - lib: some_charm.lib + - lib: some-charm.lib version: '1.2' """ ), diff --git a/tests/unit/services/test_image.py b/tests/unit/services/test_image.py new file mode 100644 index 000000000..61022d1ea --- /dev/null +++ b/tests/unit/services/test_image.py @@ -0,0 +1,111 @@ +# Copyright 2024 Canonical Ltd. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# For further info, check https://github.com/canonical/charmcraft +"""Unit tests for the Image service.""" + + +import itertools +import json +from unittest import mock + +import docker +import docker.errors +import docker.models.images +import pytest + +from charmcraft import application, const, services, utils + + +@pytest.fixture() +def mock_docker() -> mock.Mock: + return mock.Mock(spec_set=docker.DockerClient) + + +@pytest.fixture() +def mock_skopeo(fake_process) -> mock.Mock: + fake_process.register(["/skopeo", "--version"]) + return mock.Mock(wraps=utils.Skopeo(skopeo_path="/skopeo")) + + +@pytest.fixture() +def image_service(service_factory, mock_skopeo, mock_docker) -> services.ImageService: + service = services.ImageService(app=application.APP_METADATA, services=service_factory) + service._skopeo = mock_skopeo + service._docker = mock_docker + return service + + +def test_get_maybe_id_from_docker_success(image_service: services.ImageService, mock_docker): + expected = "sha256:some-sha-hash" + mock_docker.images.get.return_value = docker.models.images.Image(attrs={"Id": expected}) + + result = image_service.get_maybe_id_from_docker("some-image") + + mock_docker.images.get.assert_called_once_with("some-image") + assert result == expected + + +def test_get_maybe_id_from_docker_failure(image_service: services.ImageService, mock_docker): + mock_docker.images.get.side_effect = docker.errors.ImageNotFound("womp womp") + + assert image_service.get_maybe_id_from_docker("some-image") is None + + +@pytest.mark.parametrize("image", ["my-image"]) +@pytest.mark.parametrize("architecture", const.CharmArch) +def test_inspect_single_arch( + fake_process, image_service: services.ImageService, mock_skopeo, image: str, architecture +): + fake_process.register( + ["/skopeo", "inspect", "--raw", image], stdout=json.dumps({"raw_manifest": True}) + ) + fake_process.register( + ["/skopeo", "inspect", image], + stdout=json.dumps({"Digest": "Reader's", "Architecture": architecture}), + ) + + actual = image_service.inspect(image) + + assert actual == services.image.OCIMetadata( + path=image, digest="Reader's", architectures=[architecture] + ) + + +@pytest.mark.parametrize("image", ["my-image"]) +@pytest.mark.parametrize("architectures", itertools.product(const.CharmArch, repeat=2)) +def test_inspect_two_arch( + fake_process, image_service: services.ImageService, mock_skopeo, image: str, architectures +): + fake_process.register( + ["/skopeo", "inspect", "--raw", image], + stdout=json.dumps( + { + "manifests": [ + {"platform": {"os": "linux", "architecture": str(arch)}} + for arch in architectures + ], + } + ), + ) + fake_process.register( + ["/skopeo", "inspect", image], + stdout=json.dumps({"Digest": "Reader's", "Architecture": "amd64"}), + ) + + actual = image_service.inspect(image) + + assert actual == services.image.OCIMetadata( + path=image, digest="Reader's", architectures=list(architectures) + ) diff --git a/tests/unit/services/test_store.py b/tests/unit/services/test_store.py index ec4730d07..0090a8786 100644 --- a/tests/unit/services/test_store.py +++ b/tests/unit/services/test_store.py @@ -232,11 +232,11 @@ def test_get_credentials(monkeypatch, store): ([], []), ( [CharmLib(lib="my_charm.my_lib", version="1")], - [{"charm-name": "my_charm", "library-name": "my_lib", "api": 1}], + [{"charm-name": "my-charm", "library-name": "my_lib", "api": 1}], ), ( [CharmLib(lib="my_charm.my_lib", version="1.0")], - [{"charm-name": "my_charm", "library-name": "my_lib", "api": 1, "patch": 0}], + [{"charm-name": "my-charm", "library-name": "my_lib", "api": 1, "patch": 0}], ), ], ) diff --git a/tests/unit/utils/test_skopeo.py b/tests/unit/utils/test_skopeo.py new file mode 100644 index 000000000..af2c6668d --- /dev/null +++ b/tests/unit/utils/test_skopeo.py @@ -0,0 +1,126 @@ +# Copyright 2024 Canonical Ltd. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# For further info, check https://github.com/canonical/charmcraft +"""Unit tests for skopeo wrapper.""" + +import pathlib +import platform +from unittest import mock + +import pytest + +from charmcraft.utils.skopeo import Skopeo + +pytestmark = [ + pytest.mark.xfail( + platform.system().lower() not in ("linux", "darwin"), + reason="Don't necessarily have skopeo on non Linux/mac platforms.", + strict=False, # Allow them to pass anyway. + ), +] + +IMAGE_PATHS = [ # See: https://github.com/containers/skopeo/blob/main/docs/skopeo.1.md#image-names + "containers-storage:my/local:image", + "dir:/tmp/some-dir", + "docker://my-image:latest", + "docker-archive:/tmp/some-archive", + "docker-archive:/tmp/some-archive:latest", + "docker-daemon:sha256:f515493110d497051b4a5c4d977c2b1e7f38190def919ab22683e6785b9d5067", + "docker-daemon:ubuntu:24.04", + "oci:/tmp/some-dir:latest", + "oci-archive:my-image.tar", +] + + +@pytest.mark.parametrize("path", ["/skopeo", "/bin/skopeo"]) +def test_skopeo_path(fake_process, path): + fake_process.register([path, "--version"]) + skopeo = Skopeo(skopeo_path=path) + + assert skopeo.get_global_command() == [path] + + +def test_find_skopeo_success(fake_process): + path = "/fake/path/to/skopeo" + fake_process.register([path, "--version"]) + with mock.patch("shutil.which", return_value=path) as mock_which: + skopeo = Skopeo() + + assert skopeo.get_global_command() == [path] + mock_which.assert_called_once_with("skopeo") + + +@pytest.mark.parametrize( + ("kwargs", "expected"), + [ + pytest.param({}, [], id="empty"), + pytest.param({"insecure_policy": True}, ["--insecure-policy"], id="insecure_policy"), + pytest.param({"arch": "amd64"}, ["--override-arch", "amd64"], id="amd64"), + pytest.param({"arch": "arm64"}, ["--override-arch", "arm64"], id="arm64"), + pytest.param({"arch": "riscv64"}, ["--override-arch", "riscv64"], id="riscv64"), + pytest.param({"os": "linux"}, ["--override-os", "linux"], id="os-linux"), + pytest.param({"os": "bsd"}, ["--override-os", "bsd"], id="os-bsd"), + pytest.param( + {"tmpdir": pathlib.Path("/tmp/skopeo_tmp")}, + ["--tmpdir", "/tmp/skopeo_tmp"], + id="tmpdir", + ), + ], +) +def test_get_global_command(fake_process, kwargs, expected): + """Tests for getting the global command and arguments.""" + fake_process.register(["/skopeo", "--version"]) + skopeo = Skopeo(skopeo_path="/skopeo", **kwargs) + + assert skopeo.get_global_command() == ["/skopeo", *expected] + + +@pytest.fixture() +def fake_skopeo(fake_process): + fake_process.register(["/skopeo", "--version"]) + return Skopeo(skopeo_path="/skopeo") + + +@pytest.mark.parametrize("source_image", IMAGE_PATHS) +@pytest.mark.parametrize("destination_image", IMAGE_PATHS) +@pytest.mark.parametrize( + ("kwargs", "expected_args"), + [ + ({}, []), + ({"all_images": True}, ["--all"]), + ({"preserve_digests": True}, ["--preserve-digests"]), + ({"source_username": "user"}, ["--src-creds", "user"]), + ({"source_password": "pass"}, ["--src-password", "pass"]), + ({"source_username": "user", "source_password": "pass"}, ["--src-creds", "user:pass"]), + ({"dest_username": "user"}, ["--dest-creds", "user"]), + ({"dest_password": "pass"}, ["--dest-password", "pass"]), + ({"dest_username": "user", "dest_password": "pass"}, ["--dest-creds", "user:pass"]), + ], +) +def test_get_copy_command( + fake_process, fake_skopeo: Skopeo, source_image, destination_image, kwargs, expected_args +): + fake_process.register( + [ + *fake_skopeo.get_global_command(), + "copy", + *expected_args, + source_image, + destination_image, + ] + ) + result = fake_skopeo.copy(source_image, destination_image, **kwargs) + + result.check_returncode()