diff --git a/.github/workflows/tests.yaml b/.github/workflows/tests.yaml index 3a6335fbe..5aebb1aae 100644 --- a/.github/workflows/tests.yaml +++ b/.github/workflows/tests.yaml @@ -88,8 +88,30 @@ jobs: run: | pip install -U -r requirements-dev.txt pip install -e . + pip install tox setuptools + - name: Install external dependencies with homebrew + # This is only necessary for Linux until skopeo >= 1.11 is in repos. + # Once we're running on Noble, we can get skopeo from apt. + if: ${{ runner.os == 'Linux' || runner.os == 'macOS' }} + run: | + if [[ $(uname --kernel-name) == "Linux" ]]; then + eval "$(/home/linuxbrew/.linuxbrew/bin/brew shellenv)" + fi + brew install skopeo - name: Run tests + shell: bash run: | + if [[ $(uname --kernel-name) == "Linux" ]]; then + # Ensure the version of skopeo comes from homebrew + # This is only necessary until we move to noble. + eval "$(/home/linuxbrew/.linuxbrew/bin/brew shellenv)" + # Allow skopeo to access the contents of /run/containers + sudo chmod 777 /run/containers || true + # Add an xdg runtime dir for skopeo to look into for an auth.json file + sudo mkdir -p /run/user/$(id -u) + sudo chown $USER /run/user/$(id -u) + export XDG_RUNTIME_DIR=/run/user/$(id -u) + fi pytest -ra tests snap-build: diff --git a/CODE_OF_CONDUCT.md b/CODE_OF_CONDUCT.md index 60739500a..d3fe91379 100644 --- a/CODE_OF_CONDUCT.md +++ b/CODE_OF_CONDUCT.md @@ -6,7 +6,7 @@ In the interest of fostering an open and welcoming environment, we as contributors and maintainers pledge to make participation in our project and our community a harassment-free experience for everyone, regardless of age, body size, disability, ethnicity, sex characteristics, gender identity and -expression, level of experience, education, socio-economic status, +expression, level of experience, education, socioeconomic status, nationality, personal appearance, race, religion, or sexual identity and orientation. diff --git a/charmcraft/commands/store/__init__.py b/charmcraft/commands/store/__init__.py index bec3c13ef..ab9550444 100644 --- a/charmcraft/commands/store/__init__.py +++ b/charmcraft/commands/store/__init__.py @@ -38,7 +38,7 @@ from tabulate import tabulate from charmcraft.cmdbase import BaseCommand -from charmcraft import parts, utils +from charmcraft import errors, parts, utils from charmcraft.commands.store.registry import ImageHandler, OCIRegistry, LocalDockerdInterface from charmcraft.commands.store.store import Store, Entity @@ -1849,6 +1849,7 @@ def run(self, parsed_args): dockerd = LocalDockerdInterface() server_image_digest = None + # "Classic" method - use Docker if ":" in parsed_args.image: # the user provided a digest; check if the specific image is # already in Canonical's registry @@ -1868,14 +1869,41 @@ def run(self, parsed_args): if server_image_digest is None: if image_info is None: - raise CraftError("Image not found locally.") + emit.progress( + "Image not found locally. Passing path directly to skopeo.", + permanent=True, + ) + skopeo = utils.Skopeo() + registry_url_without_https = self.config.charmhub.registry_url[8:] + with emit.open_stream("Running Skopeo") as stream: + skopeo.copy( + parsed_args.image, + f"docker://{registry_url_without_https}/{image_name}", + dest_username=credentials.username, + dest_password=credentials.password, + all_images=True, + stdout=stream, + stderr=stream, + ) + try: + image_info = skopeo.inspect(parsed_args.image) + except errors.SubprocessError as exc: + raise errors.CraftError( + "Could not inspect OCI image.", details=f"{exc.message}\n{exc.details}" + ) + try: + server_image_digest = image_info["Digest"] + except KeyError: + raise errors.CraftError("Could not get digest for image.") - # upload it from local registry - emit.progress("Uploading from local registry.", permanent=True) - server_image_digest = ih.upload_from_local(image_info) - emit.progress( - f"Image uploaded, new remote digest: {server_image_digest}.", permanent=True - ) + else: + # upload it from local registry + emit.progress("Uploading from local registry.", permanent=True) + server_image_digest = ih.upload_from_local(image_info) + emit.progress( + f"Image uploaded, new remote digest: {server_image_digest}.", + permanent=True, + ) # all is green, get the blob to upload to Charmhub content = store.get_oci_image_blob( diff --git a/charmcraft/errors.py b/charmcraft/errors.py index 72317369a..4324ad5bc 100644 --- a/charmcraft/errors.py +++ b/charmcraft/errors.py @@ -16,6 +16,9 @@ """Charmcraft error classes.""" import io import pathlib +import shlex +import subprocess +import textwrap from typing import Iterable, Mapping from craft_cli import CraftError @@ -121,3 +124,24 @@ def __init__(self, extra_dependencies: Iterable[str]): class ExtensionError(CraftError): """Error related to extension handling.""" + + +class SubprocessError(CraftError): + """A craft-cli friendly subprocess error.""" + + @classmethod + def from_subprocess(cls, error: subprocess.CalledProcessError): + """Convert a CalledProcessError to a craft-cli error.""" + error_details = f"Full command: {shlex.join(error.cmd)}\nError text:\n" + if isinstance(error.stderr, str): + error_details += textwrap.indent(error.stderr, " ") + elif error.stderr is None: + pass + else: + stderr = error.stderr + stderr.seek(io.SEEK_SET) + error_details += textwrap.indent(stderr.read(), " ") + return cls( + f"Error while running {error.cmd[0]} (return code {error.returncode})", + details=error_details, + ) diff --git a/charmcraft/templates/init-kubernetes/src/charm.py.j2 b/charmcraft/templates/init-kubernetes/src/charm.py.j2 index 4572adf01..7c2ba6412 100755 --- a/charmcraft/templates/init-kubernetes/src/charm.py.j2 +++ b/charmcraft/templates/init-kubernetes/src/charm.py.j2 @@ -14,9 +14,9 @@ logger = logging.getLogger(__name__) class {{ class_name }}(ops.CharmBase): """Charm the application.""" - def __init__(self, *args): - super().__init__(*args) - self.framework.observe(self.on["some_container"].pebble_ready, self._on_pebble_ready) + def __init__(self, framework: ops.Framework): + super().__init__(framework) + framework.observe(self.on["some_container"].pebble_ready, self._on_pebble_ready) def _on_pebble_ready(self, event: ops.PebbleReadyEvent): """Handle pebble-ready event.""" diff --git a/charmcraft/templates/init-machine/src/charm.py.j2 b/charmcraft/templates/init-machine/src/charm.py.j2 index dd0473a3c..c57008efd 100644 --- a/charmcraft/templates/init-machine/src/charm.py.j2 +++ b/charmcraft/templates/init-machine/src/charm.py.j2 @@ -14,9 +14,9 @@ logger = logging.getLogger(__name__) class {{ class_name }}(ops.CharmBase): """Charm the application.""" - def __init__(self, *args): - super().__init__(*args) - self.framework.observe(self.on.start, self._on_start) + def __init__(self, framework: ops.Framework): + super().__init__(framework) + framework.observe(self.on.start, self._on_start) def _on_start(self, event: ops.StartEvent): """Handle start event.""" diff --git a/charmcraft/templates/init-simple/src/charm.py.j2 b/charmcraft/templates/init-simple/src/charm.py.j2 index e030a60cd..e84df9ad6 100644 --- a/charmcraft/templates/init-simple/src/charm.py.j2 +++ b/charmcraft/templates/init-simple/src/charm.py.j2 @@ -13,6 +13,7 @@ https://juju.is/docs/sdk/create-a-minimal-kubernetes-charm """ import logging +from typing import cast import ops @@ -25,10 +26,10 @@ VALID_LOG_LEVELS = ["info", "debug", "warning", "error", "critical"] class {{ class_name }}(ops.CharmBase): """Charm the service.""" - def __init__(self, *args): - super().__init__(*args) - self.framework.observe(self.on['httpbin'].pebble_ready, self._on_httpbin_pebble_ready) - self.framework.observe(self.on.config_changed, self._on_config_changed) + def __init__(self, framework: ops.Framework): + super().__init__(framework) + framework.observe(self.on["httpbin"].pebble_ready, self._on_httpbin_pebble_ready) + framework.observe(self.on.config_changed, self._on_config_changed) def _on_httpbin_pebble_ready(self, event: ops.PebbleReadyEvent): """Define and start a workload using the Pebble API. @@ -57,7 +58,7 @@ class {{ class_name }}(ops.CharmBase): Learn more about config at https://juju.is/docs/sdk/config """ # Fetch the new config value - log_level = self.model.config["log-level"].lower() + log_level = cast(str, self.model.config["log-level"]).lower() # Do some validation of the configuration option if log_level in VALID_LOG_LEVELS: diff --git a/charmcraft/utils/__init__.py b/charmcraft/utils/__init__.py index 5466e81b7..c2043f5f8 100644 --- a/charmcraft/utils/__init__.py +++ b/charmcraft/utils/__init__.py @@ -56,6 +56,7 @@ get_charm_name_from_path, get_templates_environment, ) +from charmcraft.utils.skopeo import Skopeo from charmcraft.utils.store import ChannelData, Risk from charmcraft.utils.yaml import load_yaml @@ -96,5 +97,6 @@ "find_charm_sources", "get_charm_name_from_path", "get_templates_environment", + "Skopeo", "load_yaml", ] diff --git a/charmcraft/utils/skopeo.py b/charmcraft/utils/skopeo.py new file mode 100644 index 000000000..011c823c4 --- /dev/null +++ b/charmcraft/utils/skopeo.py @@ -0,0 +1,156 @@ +# Copyright 2024 Canonical Ltd. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# For further info, check https://github.com/canonical/charmcraft +"""A wrapper around Skopeo.""" + +import io +import json +import pathlib +import shutil +import subprocess +from typing import Any, Dict, List, Optional, Sequence, Union, cast, overload + +from charmcraft import errors + + +class Skopeo: + """A class for interacting with skopeo.""" + + def __init__( + self, + *, + skopeo_path: str = "", + insecure_policy: bool = False, + arch: Union[str, None] = None, + os: Union[str, None] = None, + tmpdir: Union[pathlib.Path, None] = None, + debug: bool = False, + ) -> None: + if skopeo_path: + self._skopeo = skopeo_path + else: + self._skopeo = cast(str, shutil.which("skopeo")) + if not self._skopeo: + raise RuntimeError("Cannot find a skopeo executable.") + self._insecure_policy = insecure_policy + self.arch = arch + self.os = os + if tmpdir: + tmpdir.mkdir(parents=True, exist_ok=True) + self._tmpdir = tmpdir + self._debug = debug + + self._run_skopeo([self._skopeo, "--version"], capture_output=True, text=True) + + def get_global_command(self) -> List[str]: + """Prepare the global skopeo options.""" + command = [self._skopeo] + if self._insecure_policy: + command.append("--insecure-policy") + if self.arch: + command.extend(["--override-arch", self.arch]) + if self.os: + command.extend(["--override-os", self.os]) + if self._tmpdir: + command.extend(["--tmpdir", str(self._tmpdir)]) + if self._debug: + command.append("--debug") + return command + + def _run_skopeo(self, command: Sequence[str], **kwargs) -> subprocess.CompletedProcess: + """Run skopeo, converting the error message if necessary.""" + try: + return subprocess.run(command, check=True, **kwargs) + except subprocess.CalledProcessError as exc: + raise errors.SubprocessError.from_subprocess(exc) from exc + + def copy( + self, + source_image: str, + destination_image: str, + *, + all_images: bool = False, + preserve_digests: bool = False, + source_username: Optional[str] = None, + source_password: Optional[str] = None, + dest_username: Optional[str] = None, + dest_password: Optional[str] = None, + stdout: Union[io.FileIO, int, None] = None, + stderr: Union[io.FileIO, int, None] = None, + ) -> subprocess.CompletedProcess: + """Copy an OCI image using Skopeo.""" + command = [ + *self.get_global_command(), + "copy", + ] + if all_images: + command.append("--all") + if preserve_digests: + command.append("--preserve-digests") + if source_username and source_password: + command.extend(["--src-creds", f"{source_username}:{source_password}"]) + elif source_username: + command.extend(["--src-creds", source_username]) + elif source_password: + command.extend(["--src-password", source_password]) + if dest_username and dest_password: + command.extend(["--dest-creds", f"{dest_username}:{dest_password}"]) + elif dest_username: + command.extend(["--dest-creds", dest_username]) + elif dest_password: + command.extend(["--dest-password", dest_password]) + + command.extend([source_image, destination_image]) + + if stdout or stderr: + return self._run_skopeo(command, stdout=stdout, stderr=stderr, text=True) + return self._run_skopeo(command, capture_output=True, text=True) + + @overload + def inspect( + self, image: str, *, format_template: None = None, raw: bool = False, tags: bool = True + ) -> Dict[str, Any]: + ... + + @overload + def inspect( + self, image: str, *, format_template: str, raw: bool = False, tags: bool = True + ) -> str: + ... + + def inspect( + self, + image, + *, + format_template=None, + raw: bool = False, + tags: bool = True, + ): + """Inspect an image.""" + command = [*self.get_global_command(), "inspect"] + if format_template is not None: + command.extend(["--format", format_template]) + if raw: + command.append("--raw") + if not tags: + command.append("--no-tags") + + command.append(image) + + result = self._run_skopeo(command, capture_output=True, text=True) + + if format_template is None: + return json.loads(result.stdout) + return result.stdout diff --git a/snap/snapcraft.yaml b/snap/snapcraft.yaml index c5d8ff31c..cb683b766 100644 --- a/snap/snapcraft.yaml +++ b/snap/snapcraft.yaml @@ -134,6 +134,32 @@ parts: organize: bin/craftctl: libexec/charmcraft/craftctl + skopeo: # Copied from Rockcraft + plugin: nil + source: https://github.com/containers/skopeo.git + source-tag: v1.15.1 + override-build: | + CGO=1 go build -ldflags -linkmode=external ./cmd/skopeo + mkdir "$CRAFT_PART_INSTALL"/bin + install -m755 skopeo "$CRAFT_PART_INSTALL"/bin/skopeo + stage-packages: + - libgpgme11 + - libassuan0 + - libbtrfs0 + - libdevmapper1.02.1 + build-attributes: + - enable-patchelf + build-snaps: + - go/1.21/stable + build-packages: + - libgpgme-dev + - libassuan-dev + - libbtrfs-dev + - libdevmapper-dev + - pkg-config + organize: + bin/skopeo: libexec/charmcraft/skopeo + hooks: configure: environment: diff --git a/tests/commands/test_store_commands.py b/tests/commands/test_store_commands.py index 314a4475b..6b5b0974a 100644 --- a/tests/commands/test_store_commands.py +++ b/tests/commands/test_store_commands.py @@ -18,6 +18,7 @@ import base64 import datetime +import platform import sys import zipfile from argparse import ArgumentParser, Namespace @@ -4426,6 +4427,7 @@ def test_uploadresource_image_id_upload_from_local(emitter, store_mock, config): ) +@pytest.mark.skipif(platform.system() == "Windows", reason="No skopeo") def test_uploadresource_image_digest_missing_everywhere(emitter, store_mock, config): """Upload an oci-image resource by digest, but the image is not found remote nor locally.""" # fake credentials for the charm/resource, the final json content, and the upload result @@ -4458,7 +4460,7 @@ def test_uploadresource_image_digest_missing_everywhere(emitter, store_mock, con with pytest.raises(CraftError) as cm: UploadResourceCommand(config).run(args) - assert str(cm.value) == "Image not found locally." + assert str(cm.value).startswith("Error while running") # validate how local interfaces and store was used assert im_mock.mock_calls == [ @@ -4483,6 +4485,7 @@ def test_uploadresource_image_digest_missing_everywhere(emitter, store_mock, con ) +@pytest.mark.skipif(platform.system() == "Windows", reason="No skopeo") def test_uploadresource_image_id_missing(emitter, store_mock, config): """Upload an oci-image resource by id, but the image is not found locally.""" # fake credentials for the charm/resource, the final json content, and the upload result @@ -4512,7 +4515,7 @@ def test_uploadresource_image_id_missing(emitter, store_mock, config): with pytest.raises(CraftError) as cm: UploadResourceCommand(config).run(args) - assert str(cm.value) == "Image not found locally." + assert str(cm.value).startswith("Error while") assert dock_mock.mock_calls == [ call.get_image_info_from_id(original_image_id), diff --git a/tests/spread/store/resources/task.yaml b/tests/spread/store/resources/task.yaml index 0937b072f..f357db7fd 100644 --- a/tests/spread/store/resources/task.yaml +++ b/tests/spread/store/resources/task.yaml @@ -76,6 +76,9 @@ execute: | last_revision_created=$(echo $last_revision | jq -r .created_at) [[ $start_datetime < $last_revision_created ]] + # Check that skopeo upload-resource works. + charmcraft upload-resource $CHARM_DEFAULT_NAME example-image --image=docker://hello-world@sha256:18a657d0cc1c7d0678a3fbea8b7eb4918bba25968d3e1b0adebfa71caddbc346 + # release and check full status charmcraft release $CHARM_DEFAULT_NAME -r $last_charm_revno -c edge --resource=example-file:$last_file_revno --resource=example-image:$last_image_revno edge_release=$(charmcraft status $CHARM_DEFAULT_NAME --format=json | jq -r '.[] | select(.track=="latest") | .mappings[0].releases | .[] | select(.channel=="latest/edge")') diff --git a/tests/test_infra.py b/tests/test_infra.py index 45e40d0d9..82371ed39 100644 --- a/tests/test_infra.py +++ b/tests/test_infra.py @@ -17,12 +17,10 @@ import itertools import os import re -import subprocess -import sys import pytest -from charmcraft import __version__, main +from charmcraft import main def get_python_filepaths(*, roots=None, python_paths=None): @@ -58,15 +56,6 @@ def test_ensure_copyright(): pytest.fail(msg, pytrace=False) -@pytest.mark.skipif(sys.platform == "win32", reason="Windows not [yet] supported") -def test_setup_version(): - """Verify that setup.py is picking up the version correctly.""" - cmd = [os.path.abspath("setup.py"), "--version"] - proc = subprocess.run(cmd, stdout=subprocess.PIPE, check=True) - output = proc.stdout.decode("utf8") - assert output.strip() == __version__ - - def test_bashcompletion_all_commands(): """Verify that all commands are represented in the bash completion file.""" # get the line where all commands are specified in the completion file; this is custom diff --git a/tests/unit/utils/test_skopeo.py b/tests/unit/utils/test_skopeo.py new file mode 100644 index 000000000..af2c6668d --- /dev/null +++ b/tests/unit/utils/test_skopeo.py @@ -0,0 +1,126 @@ +# Copyright 2024 Canonical Ltd. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# For further info, check https://github.com/canonical/charmcraft +"""Unit tests for skopeo wrapper.""" + +import pathlib +import platform +from unittest import mock + +import pytest + +from charmcraft.utils.skopeo import Skopeo + +pytestmark = [ + pytest.mark.xfail( + platform.system().lower() not in ("linux", "darwin"), + reason="Don't necessarily have skopeo on non Linux/mac platforms.", + strict=False, # Allow them to pass anyway. + ), +] + +IMAGE_PATHS = [ # See: https://github.com/containers/skopeo/blob/main/docs/skopeo.1.md#image-names + "containers-storage:my/local:image", + "dir:/tmp/some-dir", + "docker://my-image:latest", + "docker-archive:/tmp/some-archive", + "docker-archive:/tmp/some-archive:latest", + "docker-daemon:sha256:f515493110d497051b4a5c4d977c2b1e7f38190def919ab22683e6785b9d5067", + "docker-daemon:ubuntu:24.04", + "oci:/tmp/some-dir:latest", + "oci-archive:my-image.tar", +] + + +@pytest.mark.parametrize("path", ["/skopeo", "/bin/skopeo"]) +def test_skopeo_path(fake_process, path): + fake_process.register([path, "--version"]) + skopeo = Skopeo(skopeo_path=path) + + assert skopeo.get_global_command() == [path] + + +def test_find_skopeo_success(fake_process): + path = "/fake/path/to/skopeo" + fake_process.register([path, "--version"]) + with mock.patch("shutil.which", return_value=path) as mock_which: + skopeo = Skopeo() + + assert skopeo.get_global_command() == [path] + mock_which.assert_called_once_with("skopeo") + + +@pytest.mark.parametrize( + ("kwargs", "expected"), + [ + pytest.param({}, [], id="empty"), + pytest.param({"insecure_policy": True}, ["--insecure-policy"], id="insecure_policy"), + pytest.param({"arch": "amd64"}, ["--override-arch", "amd64"], id="amd64"), + pytest.param({"arch": "arm64"}, ["--override-arch", "arm64"], id="arm64"), + pytest.param({"arch": "riscv64"}, ["--override-arch", "riscv64"], id="riscv64"), + pytest.param({"os": "linux"}, ["--override-os", "linux"], id="os-linux"), + pytest.param({"os": "bsd"}, ["--override-os", "bsd"], id="os-bsd"), + pytest.param( + {"tmpdir": pathlib.Path("/tmp/skopeo_tmp")}, + ["--tmpdir", "/tmp/skopeo_tmp"], + id="tmpdir", + ), + ], +) +def test_get_global_command(fake_process, kwargs, expected): + """Tests for getting the global command and arguments.""" + fake_process.register(["/skopeo", "--version"]) + skopeo = Skopeo(skopeo_path="/skopeo", **kwargs) + + assert skopeo.get_global_command() == ["/skopeo", *expected] + + +@pytest.fixture() +def fake_skopeo(fake_process): + fake_process.register(["/skopeo", "--version"]) + return Skopeo(skopeo_path="/skopeo") + + +@pytest.mark.parametrize("source_image", IMAGE_PATHS) +@pytest.mark.parametrize("destination_image", IMAGE_PATHS) +@pytest.mark.parametrize( + ("kwargs", "expected_args"), + [ + ({}, []), + ({"all_images": True}, ["--all"]), + ({"preserve_digests": True}, ["--preserve-digests"]), + ({"source_username": "user"}, ["--src-creds", "user"]), + ({"source_password": "pass"}, ["--src-password", "pass"]), + ({"source_username": "user", "source_password": "pass"}, ["--src-creds", "user:pass"]), + ({"dest_username": "user"}, ["--dest-creds", "user"]), + ({"dest_password": "pass"}, ["--dest-password", "pass"]), + ({"dest_username": "user", "dest_password": "pass"}, ["--dest-creds", "user:pass"]), + ], +) +def test_get_copy_command( + fake_process, fake_skopeo: Skopeo, source_image, destination_image, kwargs, expected_args +): + fake_process.register( + [ + *fake_skopeo.get_global_command(), + "copy", + *expected_args, + source_image, + destination_image, + ] + ) + result = fake_skopeo.copy(source_image, destination_image, **kwargs) + + result.check_returncode()