Skip to content

Commit

Permalink
feat: backport rock upload to charmcraft 2.x (#1708)
Browse files Browse the repository at this point in the history
Allows Charmcraft 2.7+ to use skopeo urls directly when uploading an
oci-image resource.

Example:

```
$ charmcraft upload-resource lengau-test-charm example-image --image docker://ghcr.io/canonical/charmed-mysql:8.0.36-22.04_edge
Image not found locally. Passing path directly to skopeo.                                         
Revision 6 created of resource 'example-image' for charm 'lengau-test-charm'.      
$
```
  • Loading branch information
lengau authored Jun 13, 2024
1 parent d4d5da9 commit 7b86722
Show file tree
Hide file tree
Showing 14 changed files with 414 additions and 34 deletions.
22 changes: 22 additions & 0 deletions .github/workflows/tests.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -88,8 +88,30 @@ jobs:
run: |
pip install -U -r requirements-dev.txt
pip install -e .
pip install tox setuptools
- name: Install external dependencies with homebrew
# This is only necessary for Linux until skopeo >= 1.11 is in repos.
# Once we're running on Noble, we can get skopeo from apt.
if: ${{ runner.os == 'Linux' || runner.os == 'macOS' }}
run: |
if [[ $(uname --kernel-name) == "Linux" ]]; then
eval "$(/home/linuxbrew/.linuxbrew/bin/brew shellenv)"
fi
brew install skopeo
- name: Run tests
shell: bash
run: |
if [[ $(uname --kernel-name) == "Linux" ]]; then
# Ensure the version of skopeo comes from homebrew
# This is only necessary until we move to noble.
eval "$(/home/linuxbrew/.linuxbrew/bin/brew shellenv)"
# Allow skopeo to access the contents of /run/containers
sudo chmod 777 /run/containers || true
# Add an xdg runtime dir for skopeo to look into for an auth.json file
sudo mkdir -p /run/user/$(id -u)
sudo chown $USER /run/user/$(id -u)
export XDG_RUNTIME_DIR=/run/user/$(id -u)
fi
pytest -ra tests
snap-build:
Expand Down
2 changes: 1 addition & 1 deletion CODE_OF_CONDUCT.md
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@ In the interest of fostering an open and welcoming environment, we as
contributors and maintainers pledge to make participation in our project and
our community a harassment-free experience for everyone, regardless of age,
body size, disability, ethnicity, sex characteristics, gender identity and
expression, level of experience, education, socio-economic status,
expression, level of experience, education, socioeconomic status,
nationality, personal appearance, race, religion, or sexual identity and
orientation.

Expand Down
44 changes: 36 additions & 8 deletions charmcraft/commands/store/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -38,7 +38,7 @@
from tabulate import tabulate

from charmcraft.cmdbase import BaseCommand
from charmcraft import parts, utils
from charmcraft import errors, parts, utils

from charmcraft.commands.store.registry import ImageHandler, OCIRegistry, LocalDockerdInterface
from charmcraft.commands.store.store import Store, Entity
Expand Down Expand Up @@ -1849,6 +1849,7 @@ def run(self, parsed_args):
dockerd = LocalDockerdInterface()

server_image_digest = None
# "Classic" method - use Docker
if ":" in parsed_args.image:
# the user provided a digest; check if the specific image is
# already in Canonical's registry
Expand All @@ -1868,14 +1869,41 @@ def run(self, parsed_args):

if server_image_digest is None:
if image_info is None:
raise CraftError("Image not found locally.")
emit.progress(
"Image not found locally. Passing path directly to skopeo.",
permanent=True,
)
skopeo = utils.Skopeo()
registry_url_without_https = self.config.charmhub.registry_url[8:]
with emit.open_stream("Running Skopeo") as stream:
skopeo.copy(
parsed_args.image,
f"docker://{registry_url_without_https}/{image_name}",
dest_username=credentials.username,
dest_password=credentials.password,
all_images=True,
stdout=stream,
stderr=stream,
)
try:
image_info = skopeo.inspect(parsed_args.image)
except errors.SubprocessError as exc:
raise errors.CraftError(
"Could not inspect OCI image.", details=f"{exc.message}\n{exc.details}"
)
try:
server_image_digest = image_info["Digest"]
except KeyError:
raise errors.CraftError("Could not get digest for image.")

# upload it from local registry
emit.progress("Uploading from local registry.", permanent=True)
server_image_digest = ih.upload_from_local(image_info)
emit.progress(
f"Image uploaded, new remote digest: {server_image_digest}.", permanent=True
)
else:
# upload it from local registry
emit.progress("Uploading from local registry.", permanent=True)
server_image_digest = ih.upload_from_local(image_info)
emit.progress(
f"Image uploaded, new remote digest: {server_image_digest}.",
permanent=True,
)

# all is green, get the blob to upload to Charmhub
content = store.get_oci_image_blob(
Expand Down
24 changes: 24 additions & 0 deletions charmcraft/errors.py
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,9 @@
"""Charmcraft error classes."""
import io
import pathlib
import shlex
import subprocess
import textwrap
from typing import Iterable, Mapping

from craft_cli import CraftError
Expand Down Expand Up @@ -121,3 +124,24 @@ def __init__(self, extra_dependencies: Iterable[str]):

class ExtensionError(CraftError):
"""Error related to extension handling."""


class SubprocessError(CraftError):
"""A craft-cli friendly subprocess error."""

@classmethod
def from_subprocess(cls, error: subprocess.CalledProcessError):
"""Convert a CalledProcessError to a craft-cli error."""
error_details = f"Full command: {shlex.join(error.cmd)}\nError text:\n"
if isinstance(error.stderr, str):
error_details += textwrap.indent(error.stderr, " ")
elif error.stderr is None:
pass
else:
stderr = error.stderr
stderr.seek(io.SEEK_SET)
error_details += textwrap.indent(stderr.read(), " ")
return cls(
f"Error while running {error.cmd[0]} (return code {error.returncode})",
details=error_details,
)
6 changes: 3 additions & 3 deletions charmcraft/templates/init-kubernetes/src/charm.py.j2
Original file line number Diff line number Diff line change
Expand Up @@ -14,9 +14,9 @@ logger = logging.getLogger(__name__)
class {{ class_name }}(ops.CharmBase):
"""Charm the application."""

def __init__(self, *args):
super().__init__(*args)
self.framework.observe(self.on["some_container"].pebble_ready, self._on_pebble_ready)
def __init__(self, framework: ops.Framework):
super().__init__(framework)
framework.observe(self.on["some_container"].pebble_ready, self._on_pebble_ready)

def _on_pebble_ready(self, event: ops.PebbleReadyEvent):
"""Handle pebble-ready event."""
Expand Down
6 changes: 3 additions & 3 deletions charmcraft/templates/init-machine/src/charm.py.j2
Original file line number Diff line number Diff line change
Expand Up @@ -14,9 +14,9 @@ logger = logging.getLogger(__name__)
class {{ class_name }}(ops.CharmBase):
"""Charm the application."""

def __init__(self, *args):
super().__init__(*args)
self.framework.observe(self.on.start, self._on_start)
def __init__(self, framework: ops.Framework):
super().__init__(framework)
framework.observe(self.on.start, self._on_start)

def _on_start(self, event: ops.StartEvent):
"""Handle start event."""
Expand Down
11 changes: 6 additions & 5 deletions charmcraft/templates/init-simple/src/charm.py.j2
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,7 @@ https://juju.is/docs/sdk/create-a-minimal-kubernetes-charm
"""

import logging
from typing import cast

import ops

Expand All @@ -25,10 +26,10 @@ VALID_LOG_LEVELS = ["info", "debug", "warning", "error", "critical"]
class {{ class_name }}(ops.CharmBase):
"""Charm the service."""

def __init__(self, *args):
super().__init__(*args)
self.framework.observe(self.on['httpbin'].pebble_ready, self._on_httpbin_pebble_ready)
self.framework.observe(self.on.config_changed, self._on_config_changed)
def __init__(self, framework: ops.Framework):
super().__init__(framework)
framework.observe(self.on["httpbin"].pebble_ready, self._on_httpbin_pebble_ready)
framework.observe(self.on.config_changed, self._on_config_changed)

def _on_httpbin_pebble_ready(self, event: ops.PebbleReadyEvent):
"""Define and start a workload using the Pebble API.
Expand Down Expand Up @@ -57,7 +58,7 @@ class {{ class_name }}(ops.CharmBase):
Learn more about config at https://juju.is/docs/sdk/config
"""
# Fetch the new config value
log_level = self.model.config["log-level"].lower()
log_level = cast(str, self.model.config["log-level"]).lower()

# Do some validation of the configuration option
if log_level in VALID_LOG_LEVELS:
Expand Down
2 changes: 2 additions & 0 deletions charmcraft/utils/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -56,6 +56,7 @@
get_charm_name_from_path,
get_templates_environment,
)
from charmcraft.utils.skopeo import Skopeo
from charmcraft.utils.store import ChannelData, Risk
from charmcraft.utils.yaml import load_yaml

Expand Down Expand Up @@ -96,5 +97,6 @@
"find_charm_sources",
"get_charm_name_from_path",
"get_templates_environment",
"Skopeo",
"load_yaml",
]
156 changes: 156 additions & 0 deletions charmcraft/utils/skopeo.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,156 @@
# Copyright 2024 Canonical Ltd.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
# For further info, check https://github.com/canonical/charmcraft
"""A wrapper around Skopeo."""

import io
import json
import pathlib
import shutil
import subprocess
from typing import Any, Dict, List, Optional, Sequence, Union, cast, overload

from charmcraft import errors


class Skopeo:
"""A class for interacting with skopeo."""

def __init__(
self,
*,
skopeo_path: str = "",
insecure_policy: bool = False,
arch: Union[str, None] = None,
os: Union[str, None] = None,
tmpdir: Union[pathlib.Path, None] = None,
debug: bool = False,
) -> None:
if skopeo_path:
self._skopeo = skopeo_path
else:
self._skopeo = cast(str, shutil.which("skopeo"))
if not self._skopeo:
raise RuntimeError("Cannot find a skopeo executable.")
self._insecure_policy = insecure_policy
self.arch = arch
self.os = os
if tmpdir:
tmpdir.mkdir(parents=True, exist_ok=True)
self._tmpdir = tmpdir
self._debug = debug

self._run_skopeo([self._skopeo, "--version"], capture_output=True, text=True)

def get_global_command(self) -> List[str]:
"""Prepare the global skopeo options."""
command = [self._skopeo]
if self._insecure_policy:
command.append("--insecure-policy")
if self.arch:
command.extend(["--override-arch", self.arch])
if self.os:
command.extend(["--override-os", self.os])
if self._tmpdir:
command.extend(["--tmpdir", str(self._tmpdir)])
if self._debug:
command.append("--debug")
return command

def _run_skopeo(self, command: Sequence[str], **kwargs) -> subprocess.CompletedProcess:
"""Run skopeo, converting the error message if necessary."""
try:
return subprocess.run(command, check=True, **kwargs)
except subprocess.CalledProcessError as exc:
raise errors.SubprocessError.from_subprocess(exc) from exc

def copy(
self,
source_image: str,
destination_image: str,
*,
all_images: bool = False,
preserve_digests: bool = False,
source_username: Optional[str] = None,
source_password: Optional[str] = None,
dest_username: Optional[str] = None,
dest_password: Optional[str] = None,
stdout: Union[io.FileIO, int, None] = None,
stderr: Union[io.FileIO, int, None] = None,
) -> subprocess.CompletedProcess:
"""Copy an OCI image using Skopeo."""
command = [
*self.get_global_command(),
"copy",
]
if all_images:
command.append("--all")
if preserve_digests:
command.append("--preserve-digests")
if source_username and source_password:
command.extend(["--src-creds", f"{source_username}:{source_password}"])
elif source_username:
command.extend(["--src-creds", source_username])
elif source_password:
command.extend(["--src-password", source_password])
if dest_username and dest_password:
command.extend(["--dest-creds", f"{dest_username}:{dest_password}"])
elif dest_username:
command.extend(["--dest-creds", dest_username])
elif dest_password:
command.extend(["--dest-password", dest_password])

command.extend([source_image, destination_image])

if stdout or stderr:
return self._run_skopeo(command, stdout=stdout, stderr=stderr, text=True)
return self._run_skopeo(command, capture_output=True, text=True)

@overload
def inspect(
self, image: str, *, format_template: None = None, raw: bool = False, tags: bool = True
) -> Dict[str, Any]:
...

@overload
def inspect(
self, image: str, *, format_template: str, raw: bool = False, tags: bool = True
) -> str:
...

def inspect(
self,
image,
*,
format_template=None,
raw: bool = False,
tags: bool = True,
):
"""Inspect an image."""
command = [*self.get_global_command(), "inspect"]
if format_template is not None:
command.extend(["--format", format_template])
if raw:
command.append("--raw")
if not tags:
command.append("--no-tags")

command.append(image)

result = self._run_skopeo(command, capture_output=True, text=True)

if format_template is None:
return json.loads(result.stdout)
return result.stdout
Loading

0 comments on commit 7b86722

Please sign in to comment.