From e15c246550cbbe3325b6b8411c8f8a1a8087d744 Mon Sep 17 00:00:00 2001 From: Maksym Pavlenko Date: Wed, 20 Sep 2023 12:12:37 -0700 Subject: [PATCH 1/4] Move CRI image service into a separate plugin Signed-off-by: Maksym Pavlenko Signed-off-by: Abel Feng --- contrib/fuzz/cri_sbserver_fuzzer.go | 46 --------- contrib/fuzz/cri_server_fuzzer.go | 8 +- pkg/cri/cri.go | 55 +++-------- pkg/cri/server/base/cri_base.go | 121 ++++++++++++++++++++++++ pkg/cri/server/container_status_test.go | 2 + pkg/cri/server/images/service.go | 65 ++++++++++++- pkg/cri/server/service.go | 32 +------ 7 files changed, 209 insertions(+), 120 deletions(-) delete mode 100644 contrib/fuzz/cri_sbserver_fuzzer.go create mode 100644 pkg/cri/server/base/cri_base.go diff --git a/contrib/fuzz/cri_sbserver_fuzzer.go b/contrib/fuzz/cri_sbserver_fuzzer.go deleted file mode 100644 index b6b39201c3cf..000000000000 --- a/contrib/fuzz/cri_sbserver_fuzzer.go +++ /dev/null @@ -1,46 +0,0 @@ -//go:build gofuzz - -/* - Copyright The containerd Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package fuzz - -import ( - fuzz "github.com/AdaLogics/go-fuzz-headers" - - containerd "github.com/containerd/containerd/v2/client" - criconfig "github.com/containerd/containerd/v2/pkg/cri/config" - "github.com/containerd/containerd/v2/pkg/cri/server" -) - -func FuzzCRISandboxServer(data []byte) int { - initDaemon.Do(startDaemon) - - f := fuzz.NewConsumer(data) - - client, err := containerd.New(defaultAddress) - if err != nil { - return 0 - } - defer client.Close() - - c, err := server.NewCRIService(criconfig.Config{}, client, nil) - if err != nil { - panic(err) - } - - return fuzzCRI(f, c) -} diff --git a/contrib/fuzz/cri_server_fuzzer.go b/contrib/fuzz/cri_server_fuzzer.go index f43563e86e33..7deded3aa126 100644 --- a/contrib/fuzz/cri_server_fuzzer.go +++ b/contrib/fuzz/cri_server_fuzzer.go @@ -24,6 +24,7 @@ import ( containerd "github.com/containerd/containerd/v2/client" criconfig "github.com/containerd/containerd/v2/pkg/cri/config" "github.com/containerd/containerd/v2/pkg/cri/server" + "github.com/containerd/containerd/v2/pkg/cri/server/images" ) func FuzzCRIServer(data []byte) int { @@ -37,7 +38,12 @@ func FuzzCRIServer(data []byte) int { } defer client.Close() - c, err := server.NewCRIService(criconfig.Config{}, client, nil) + imageService, err := images.NewService(criconfig.Config{}, client) + if err != nil { + panic(err) + } + + c, err := server.NewCRIService(criconfig.Config{}, imageService, client, nil) if err != nil { panic(err) } diff --git a/pkg/cri/cri.go b/pkg/cri/cri.go index 819e2b8e4a56..ef019148db68 100644 --- a/pkg/cri/cri.go +++ b/pkg/cri/cri.go @@ -17,21 +17,19 @@ package cri import ( - "flag" "fmt" - "path/filepath" "github.com/containerd/log" "github.com/containerd/plugin" "github.com/containerd/plugin/registry" - imagespec "github.com/opencontainers/image-spec/specs-go/v1" - "k8s.io/klog/v2" containerd "github.com/containerd/containerd/v2/client" criconfig "github.com/containerd/containerd/v2/pkg/cri/config" "github.com/containerd/containerd/v2/pkg/cri/constants" "github.com/containerd/containerd/v2/pkg/cri/nri" "github.com/containerd/containerd/v2/pkg/cri/server" + "github.com/containerd/containerd/v2/pkg/cri/server/base" + "github.com/containerd/containerd/v2/pkg/cri/server/images" nriservice "github.com/containerd/containerd/v2/pkg/nri" "github.com/containerd/containerd/v2/platforms" "github.com/containerd/containerd/v2/plugins" @@ -43,7 +41,7 @@ func init() { config := criconfig.DefaultConfig() registry.Register(&plugin.Registration{ Type: plugins.GRPCPlugin, - ID: "cri", + ID: "cri-runtime-service", Config: &config, Requires: []plugin.Type{ plugins.EventPlugin, @@ -57,8 +55,6 @@ func init() { } func initCRIService(ic *plugin.InitContext) (interface{}, error) { - ic.Meta.Platforms = []imagespec.Platform{platforms.DefaultSpec()} - ic.Meta.Exports = map[string]string{"CRIVersion": constants.CRIVersion} ctx := ic.Context pluginConfig := ic.Config.(*criconfig.PluginConfig) if warnings, err := criconfig.ValidatePluginConfig(ctx, pluginConfig); err != nil { @@ -74,18 +70,19 @@ func initCRIService(ic *plugin.InitContext) (interface{}, error) { } } - c := criconfig.Config{ - PluginConfig: *pluginConfig, - ContainerdRootDir: filepath.Dir(ic.Properties[plugins.PropertyRootDir]), - ContainerdEndpoint: ic.Properties[plugins.PropertyGRPCAddress], - RootDir: ic.Properties[plugins.PropertyRootDir], - StateDir: ic.Properties[plugins.PropertyStateDir], + // Get base CRI dependencies. + criBasePlugin, err := ic.GetByID(plugins.GRPCPlugin, "cri") + if err != nil { + return nil, fmt.Errorf("unable to load CRI service base dependencies: %w", err) } - log.G(ctx).Infof("Start cri plugin with config %+v", c) + criBase := criBasePlugin.(*base.CRIBase) - if err := setGLogLevel(); err != nil { - return nil, fmt.Errorf("failed to set glog level: %w", err) + // Get image service. + criImagePlugin, err := ic.GetByID(plugins.GRPCPlugin, "cri-image-service") + if err != nil { + return nil, fmt.Errorf("unable to load CRI image service plugin dependency: %w", err) } + imageService := criImagePlugin.(*images.CRIImageService) log.G(ctx).Info("Connect containerd service") client, err := containerd.New( @@ -95,11 +92,8 @@ func initCRIService(ic *plugin.InitContext) (interface{}, error) { containerd.WithInMemoryServices(ic), containerd.WithInMemorySandboxControllers(ic), ) - if err != nil { - return nil, fmt.Errorf("failed to create containerd client: %w", err) - } - s, err := server.NewCRIService(c, client, getNRIAPI(ic)) + s, err := server.NewCRIService(criBase.Config, imageService, client, getNRIAPI(ic)) if err != nil { return nil, fmt.Errorf("failed to create CRI service: %w", err) } @@ -116,27 +110,6 @@ func initCRIService(ic *plugin.InitContext) (interface{}, error) { return s, nil } -// Set glog level. -func setGLogLevel() error { - l := log.GetLevel() - fs := flag.NewFlagSet("klog", flag.PanicOnError) - klog.InitFlags(fs) - if err := fs.Set("logtostderr", "true"); err != nil { - return err - } - switch l { - case log.TraceLevel: - return fs.Set("v", "5") - case log.DebugLevel: - return fs.Set("v", "4") - case log.InfoLevel: - return fs.Set("v", "2") - default: - // glog doesn't support other filters. Defaults to v=0. - } - return nil -} - // Get the NRI plugin, and set up our NRI API for it. func getNRIAPI(ic *plugin.InitContext) *nri.API { const ( diff --git a/pkg/cri/server/base/cri_base.go b/pkg/cri/server/base/cri_base.go new file mode 100644 index 000000000000..ca2f7a3a9552 --- /dev/null +++ b/pkg/cri/server/base/cri_base.go @@ -0,0 +1,121 @@ +/* + Copyright The containerd Authors. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +package base + +import ( + "flag" + "fmt" + "path/filepath" + + "github.com/containerd/log" + imagespec "github.com/opencontainers/image-spec/specs-go/v1" + "k8s.io/klog/v2" + + "github.com/containerd/containerd" + criconfig "github.com/containerd/containerd/pkg/cri/config" + "github.com/containerd/containerd/pkg/cri/constants" + "github.com/containerd/containerd/platforms" + "github.com/containerd/containerd/plugin" + "github.com/containerd/containerd/plugin/registry" + "github.com/containerd/containerd/plugins" +) + +// CRIBase contains common dependencies for CRI's runtime, image, and podsandbox services. +type CRIBase struct { + // Config contains all configurations. + Config criconfig.Config + // Client is an instance of the containerd client + Client *containerd.Client +} + +func init() { + config := criconfig.DefaultConfig() + + // Base plugin that other CRI services depend on. + registry.Register(&plugin.Registration{ + Type: plugins.GRPCPlugin, + ID: "cri", + Config: &config, + Requires: []plugin.Type{ + plugins.EventPlugin, + plugins.ServicePlugin, + plugins.NRIApiPlugin, + }, + InitFn: initCRIBase, + }) +} + +func initCRIBase(ic *plugin.InitContext) (interface{}, error) { + ic.Meta.Platforms = []imagespec.Platform{platforms.DefaultSpec()} + ic.Meta.Exports = map[string]string{"CRIVersion": constants.CRIVersion} + ctx := ic.Context + pluginConfig := ic.Config.(*criconfig.PluginConfig) + if err := criconfig.ValidatePluginConfig(ctx, pluginConfig); err != nil { + return nil, fmt.Errorf("invalid plugin config: %w", err) + } + + c := criconfig.Config{ + PluginConfig: *pluginConfig, + ContainerdRootDir: filepath.Dir(ic.Properties[plugins.PropertyRootDir]), + ContainerdEndpoint: ic.Properties[plugins.PropertyGRPCAddress], + RootDir: ic.Properties[plugins.PropertyRootDir], + StateDir: ic.Properties[plugins.PropertyStateDir], + } + + log.G(ctx).Infof("Start cri plugin with config %+v", c) + + if err := setGLogLevel(); err != nil { + return nil, fmt.Errorf("failed to set glog level: %w", err) + } + + log.G(ctx).Info("Connect containerd service") + client, err := containerd.New( + "", + containerd.WithDefaultNamespace(constants.K8sContainerdNamespace), + containerd.WithDefaultPlatform(platforms.Default()), + containerd.WithInMemoryServices(ic), + ) + if err != nil { + return nil, fmt.Errorf("failed to create containerd client: %w", err) + } + + return &CRIBase{ + Config: c, + Client: client, + }, nil +} + +// Set glog level. +func setGLogLevel() error { + l := log.GetLevel() + fs := flag.NewFlagSet("klog", flag.PanicOnError) + klog.InitFlags(fs) + if err := fs.Set("logtostderr", "true"); err != nil { + return err + } + switch l { + case log.TraceLevel: + return fs.Set("v", "5") + case log.DebugLevel: + return fs.Set("v", "4") + case log.InfoLevel: + return fs.Set("v", "2") + default: + // glog doesn't support other filters. Defaults to v=0. + } + return nil +} diff --git a/pkg/cri/server/container_status_test.go b/pkg/cri/server/container_status_test.go index 517cbbeacc42..289cf1080929 100644 --- a/pkg/cri/server/container_status_test.go +++ b/pkg/cri/server/container_status_test.go @@ -286,6 +286,8 @@ func (s *fakeImageService) LocalResolve(refOrID string) (imagestore.Image, error return imagestore.Image{}, errors.New("not implemented") } +func (s *fakeImageService) ImageFSPaths() map[string]string { return make(map[string]string) } + func patchExceptedWithState(expected *runtime.ContainerStatus, state runtime.ContainerState) { expected.State = state switch state { diff --git a/pkg/cri/server/images/service.go b/pkg/cri/server/images/service.go index bbf7b6f1b72f..7373c0de560c 100644 --- a/pkg/cri/server/images/service.go +++ b/pkg/cri/server/images/service.go @@ -19,21 +19,50 @@ package images import ( "context" "fmt" + "path/filepath" "time" + "github.com/containerd/log" + "github.com/containerd/plugin" + "github.com/containerd/plugin/registry" + docker "github.com/distribution/reference" + imagedigest "github.com/opencontainers/go-digest" + containerd "github.com/containerd/containerd/v2/client" criconfig "github.com/containerd/containerd/v2/pkg/cri/config" + "github.com/containerd/containerd/v2/pkg/cri/server/base" imagestore "github.com/containerd/containerd/v2/pkg/cri/store/image" snapshotstore "github.com/containerd/containerd/v2/pkg/cri/store/snapshot" ctrdutil "github.com/containerd/containerd/v2/pkg/cri/util" "github.com/containerd/containerd/v2/pkg/kmutex" "github.com/containerd/containerd/v2/platforms" + "github.com/containerd/containerd/v2/plugins" snapshot "github.com/containerd/containerd/v2/snapshots" - "github.com/containerd/log" - docker "github.com/distribution/reference" - imagedigest "github.com/opencontainers/go-digest" ) +func init() { + registry.Register(&plugin.Registration{ + Type: plugins.GRPCPlugin, + ID: "cri-image-service", + InitFn: func(ic *plugin.InitContext) (interface{}, error) { + // Get base CRI dependencies. + criPlugin, err := ic.GetByID(plugins.GRPCPlugin, "cri") + if err != nil { + return nil, fmt.Errorf("unable to load CRI service base dependencies: %w", err) + } + + cri := criPlugin.(*base.CRIBase) + + service, err := NewService(cri.Config, cri.Client) + if err != nil { + return nil, fmt.Errorf("failed to create image service: %w", err) + } + + return service, nil + }, + }) +} + type CRIImageService struct { // config contains all configurations. config criconfig.Config @@ -51,7 +80,25 @@ type CRIImageService struct { unpackDuplicationSuppressor kmutex.KeyedLocker } -func NewService(config criconfig.Config, imageFSPaths map[string]string, client *containerd.Client) (*CRIImageService, error) { +func NewService(config criconfig.Config, client *containerd.Client) (*CRIImageService, error) { + if client.SnapshotService(config.ContainerdConfig.Snapshotter) == nil { + return nil, fmt.Errorf("failed to find snapshotter %q", config.ContainerdConfig.Snapshotter) + } + + imageFSPaths := map[string]string{} + for _, ociRuntime := range config.ContainerdConfig.Runtimes { + // Can not use `c.RuntimeSnapshotter() yet, so hard-coding here.` + snapshotter := ociRuntime.Snapshotter + if snapshotter != "" { + imageFSPaths[snapshotter] = imageFSPath(config.ContainerdRootDir, snapshotter) + log.L.Infof("Get image filesystem path %q for snapshotter %q", imageFSPaths[snapshotter], snapshotter) + } + } + + snapshotter := config.ContainerdConfig.Snapshotter + imageFSPaths[snapshotter] = imageFSPath(config.ContainerdRootDir, snapshotter) + log.L.Infof("Get image filesystem path %q for snapshotter %q", imageFSPaths[snapshotter], snapshotter) + svc := CRIImageService{ config: config, client: client, @@ -94,6 +141,12 @@ func NewService(config criconfig.Config, imageFSPaths map[string]string, client return &svc, nil } +// imageFSPath returns containerd image filesystem path. +// Note that if containerd changes directory layout, we also needs to change this. +func imageFSPath(rootDir, snapshotter string) string { + return filepath.Join(rootDir, plugins.SnapshotPlugin.String()+"."+snapshotter) +} + // LocalResolve resolves image reference locally and returns corresponding image metadata. It // returns errdefs.ErrNotFound if the reference doesn't exist. func (c *CRIImageService) LocalResolve(refOrID string) (imagestore.Image, error) { @@ -148,3 +201,7 @@ func (c *CRIImageService) GetSnapshot(key, snapshotter string) (snapshotstore.Sn } return c.snapshotStore.Get(snapshotKey) } + +func (c *CRIImageService) ImageFSPaths() map[string]string { + return c.imageFSPaths +} diff --git a/pkg/cri/server/service.go b/pkg/cri/server/service.go index ab073e0002cc..8fb40e0e820c 100644 --- a/pkg/cri/server/service.go +++ b/pkg/cri/server/service.go @@ -38,7 +38,6 @@ import ( criconfig "github.com/containerd/containerd/v2/pkg/cri/config" "github.com/containerd/containerd/v2/pkg/cri/instrument" "github.com/containerd/containerd/v2/pkg/cri/nri" - "github.com/containerd/containerd/v2/pkg/cri/server/images" "github.com/containerd/containerd/v2/pkg/cri/server/podsandbox" containerstore "github.com/containerd/containerd/v2/pkg/cri/store/container" imagestore "github.com/containerd/containerd/v2/pkg/cri/store/image" @@ -83,6 +82,8 @@ type imageService interface { GetSnapshot(key, snapshotter string) (snapshotstore.Snapshot, error) LocalResolve(refOrID string) (imagestore.Image, error) + + ImageFSPaths() map[string]string } // criService implements CRIService. @@ -133,39 +134,14 @@ type criService struct { } // NewCRIService returns a new instance of CRIService -func NewCRIService(config criconfig.Config, client *containerd.Client, nri *nri.API) (CRIService, error) { +func NewCRIService(config criconfig.Config, imageService imageService, client *containerd.Client, nri *nri.API) (CRIService, error) { var err error labels := label.NewStore() - - if client.SnapshotService(config.ContainerdConfig.Snapshotter) == nil { - return nil, fmt.Errorf("failed to find snapshotter %q", config.ContainerdConfig.Snapshotter) - } - - imageFSPaths := map[string]string{} - for _, ociRuntime := range config.ContainerdConfig.Runtimes { - // Can not use `c.RuntimeSnapshotter() yet, so hard-coding here.` - snapshotter := ociRuntime.Snapshotter - if snapshotter != "" { - imageFSPaths[snapshotter] = imageFSPath(config.ContainerdRootDir, snapshotter) - log.L.Infof("Get image filesystem path %q for snapshotter %q", imageFSPaths[snapshotter], snapshotter) - } - } - - snapshotter := config.ContainerdConfig.Snapshotter - imageFSPaths[snapshotter] = imageFSPath(config.ContainerdRootDir, snapshotter) - log.L.Infof("Get image filesystem path %q for snapshotter %q", imageFSPaths[snapshotter], snapshotter) - - // TODO: expose this as a separate containerd plugin. - imageService, err := images.NewService(config, imageFSPaths, client) - if err != nil { - return nil, fmt.Errorf("unable to create CRI image service: %w", err) - } - c := &criService{ imageService: imageService, config: config, client: client, - imageFSPaths: imageFSPaths, + imageFSPaths: imageService.ImageFSPaths(), os: osinterface.RealOS{}, sandboxStore: sandboxstore.NewStore(labels), containerStore: containerstore.NewStore(labels), From c8012b6d74d83be2bd2fac69de6239b3fac4c00a Mon Sep 17 00:00:00 2001 From: Abel Feng Date: Mon, 23 Oct 2023 11:19:29 +0800 Subject: [PATCH 2/4] sandbox: make a clear dependency of cri plugins Signed-off-by: Abel Feng --- contrib/fuzz/cri_server_fuzzer.go | 13 ++- integration/image_pull_timeout_test.go | 27 ++++-- pkg/cri/cri.go | 40 ++++----- pkg/cri/server/base/cri_base.go | 104 ++++++++++++++++++------ pkg/cri/server/base/cri_base_test.go | 60 ++++++++++++++ pkg/cri/server/images/service.go | 24 +++++- pkg/cri/server/podsandbox/controller.go | 43 +++++++--- pkg/cri/server/service.go | 65 ++------------- pkg/cri/server/service_test.go | 97 ---------------------- plugins/types.go | 2 + 10 files changed, 244 insertions(+), 231 deletions(-) create mode 100644 pkg/cri/server/base/cri_base_test.go diff --git a/contrib/fuzz/cri_server_fuzzer.go b/contrib/fuzz/cri_server_fuzzer.go index 7deded3aa126..64a160b43746 100644 --- a/contrib/fuzz/cri_server_fuzzer.go +++ b/contrib/fuzz/cri_server_fuzzer.go @@ -22,8 +22,10 @@ import ( fuzz "github.com/AdaLogics/go-fuzz-headers" containerd "github.com/containerd/containerd/v2/client" + "github.com/containerd/containerd/v2/oci" criconfig "github.com/containerd/containerd/v2/pkg/cri/config" "github.com/containerd/containerd/v2/pkg/cri/server" + "github.com/containerd/containerd/v2/pkg/cri/server/base" "github.com/containerd/containerd/v2/pkg/cri/server/images" ) @@ -38,12 +40,19 @@ func FuzzCRIServer(data []byte) int { } defer client.Close() - imageService, err := images.NewService(criconfig.Config{}, client) + config := criconfig.Config{} + + criBase := &base.CRIBase{ + Config: config, + BaseOCISpecs: map[string]*oci.Spec{}, + } + + imageService, err := images.NewService(config, client) if err != nil { panic(err) } - c, err := server.NewCRIService(criconfig.Config{}, imageService, client, nil) + c, err := server.NewCRIService(criBase, imageService, client, nil) if err != nil { panic(err) } diff --git a/integration/image_pull_timeout_test.go b/integration/image_pull_timeout_test.go index 9f9966db26d7..e8b6e1590ea3 100644 --- a/integration/image_pull_timeout_test.go +++ b/integration/image_pull_timeout_test.go @@ -33,17 +33,21 @@ import ( "testing" "time" + "github.com/containerd/log" + "github.com/containerd/log/logtest" + ocispec "github.com/opencontainers/image-spec/specs-go/v1" + "github.com/stretchr/testify/assert" + runtimeapi "k8s.io/cri-api/pkg/apis/runtime/v1" + containerd "github.com/containerd/containerd/v2/client" "github.com/containerd/containerd/v2/content" "github.com/containerd/containerd/v2/leases" "github.com/containerd/containerd/v2/namespaces" + "github.com/containerd/containerd/v2/oci" criconfig "github.com/containerd/containerd/v2/pkg/cri/config" criserver "github.com/containerd/containerd/v2/pkg/cri/server" - "github.com/containerd/log" - "github.com/containerd/log/logtest" - ocispec "github.com/opencontainers/image-spec/specs-go/v1" - "github.com/stretchr/testify/assert" - runtimeapi "k8s.io/cri-api/pkg/apis/runtime/v1" + "github.com/containerd/containerd/v2/pkg/cri/server/base" + "github.com/containerd/containerd/v2/pkg/cri/server/images" ) var ( @@ -465,5 +469,16 @@ func initLocalCRIPlugin(client *containerd.Client, tmpDir string, registryCfg cr RootDir: filepath.Join(criWorkDir, "root"), StateDir: filepath.Join(criWorkDir, "state"), } - return criserver.NewCRIService(cfg, client, nil) + + criBase := &base.CRIBase{ + Config: cfg, + BaseOCISpecs: map[string]*oci.Spec{}, + } + + imageService, err := images.NewService(cfg, client) + if err != nil { + panic(err) + } + + return criserver.NewCRIService(criBase, imageService, client, nil) } diff --git a/pkg/cri/cri.go b/pkg/cri/cri.go index ef019148db68..6dab36c5a6b5 100644 --- a/pkg/cri/cri.go +++ b/pkg/cri/cri.go @@ -24,7 +24,6 @@ import ( "github.com/containerd/plugin/registry" containerd "github.com/containerd/containerd/v2/client" - criconfig "github.com/containerd/containerd/v2/pkg/cri/config" "github.com/containerd/containerd/v2/pkg/cri/constants" "github.com/containerd/containerd/v2/pkg/cri/nri" "github.com/containerd/containerd/v2/pkg/cri/server" @@ -33,22 +32,23 @@ import ( nriservice "github.com/containerd/containerd/v2/pkg/nri" "github.com/containerd/containerd/v2/platforms" "github.com/containerd/containerd/v2/plugins" - "github.com/containerd/containerd/v2/services/warning" ) // Register CRI service plugin func init() { - config := criconfig.DefaultConfig() + registry.Register(&plugin.Registration{ - Type: plugins.GRPCPlugin, - ID: "cri-runtime-service", - Config: &config, + Type: plugins.GRPCPlugin, + ID: "cri", Requires: []plugin.Type{ + plugins.CRIImagePlugin, + plugins.InternalPlugin, + plugins.SandboxControllerPlugin, + plugins.NRIApiPlugin, plugins.EventPlugin, plugins.ServicePlugin, - plugins.NRIApiPlugin, - plugins.WarningPlugin, - plugins.SandboxControllerPlugin, + plugins.LeasePlugin, + plugins.SandboxStorePlugin, }, InitFn: initCRIService, }) @@ -56,29 +56,16 @@ func init() { func initCRIService(ic *plugin.InitContext) (interface{}, error) { ctx := ic.Context - pluginConfig := ic.Config.(*criconfig.PluginConfig) - if warnings, err := criconfig.ValidatePluginConfig(ctx, pluginConfig); err != nil { - return nil, fmt.Errorf("invalid plugin config: %w", err) - } else if len(warnings) > 0 { - ws, err := ic.GetSingle(plugins.WarningPlugin) - if err != nil { - return nil, err - } - warn := ws.(warning.Service) - for _, w := range warnings { - warn.Emit(ctx, w) - } - } // Get base CRI dependencies. - criBasePlugin, err := ic.GetByID(plugins.GRPCPlugin, "cri") + criBasePlugin, err := ic.GetByID(plugins.InternalPlugin, "cri") if err != nil { return nil, fmt.Errorf("unable to load CRI service base dependencies: %w", err) } criBase := criBasePlugin.(*base.CRIBase) // Get image service. - criImagePlugin, err := ic.GetByID(plugins.GRPCPlugin, "cri-image-service") + criImagePlugin, err := ic.GetByID(plugins.CRIImagePlugin, "cri-image-service") if err != nil { return nil, fmt.Errorf("unable to load CRI image service plugin dependency: %w", err) } @@ -92,8 +79,11 @@ func initCRIService(ic *plugin.InitContext) (interface{}, error) { containerd.WithInMemoryServices(ic), containerd.WithInMemorySandboxControllers(ic), ) + if err != nil { + return nil, fmt.Errorf("failed to create containerd client: %w", err) + } - s, err := server.NewCRIService(criBase.Config, imageService, client, getNRIAPI(ic)) + s, err := server.NewCRIService(criBase, imageService, client, getNRIAPI(ic)) if err != nil { return nil, fmt.Errorf("failed to create CRI service: %w", err) } diff --git a/pkg/cri/server/base/cri_base.go b/pkg/cri/server/base/cri_base.go index ca2f7a3a9552..553393774030 100644 --- a/pkg/cri/server/base/cri_base.go +++ b/pkg/cri/server/base/cri_base.go @@ -17,29 +17,34 @@ package base import ( + "context" + "encoding/json" "flag" "fmt" + "os" "path/filepath" "github.com/containerd/log" + "github.com/containerd/plugin" + "github.com/containerd/plugin/registry" imagespec "github.com/opencontainers/image-spec/specs-go/v1" "k8s.io/klog/v2" - "github.com/containerd/containerd" - criconfig "github.com/containerd/containerd/pkg/cri/config" - "github.com/containerd/containerd/pkg/cri/constants" - "github.com/containerd/containerd/platforms" - "github.com/containerd/containerd/plugin" - "github.com/containerd/containerd/plugin/registry" - "github.com/containerd/containerd/plugins" + "github.com/containerd/containerd/v2/oci" + criconfig "github.com/containerd/containerd/v2/pkg/cri/config" + "github.com/containerd/containerd/v2/pkg/cri/constants" + "github.com/containerd/containerd/v2/platforms" + "github.com/containerd/containerd/v2/plugins" + srvconfig "github.com/containerd/containerd/v2/services/server/config" + "github.com/containerd/containerd/v2/services/warning" ) // CRIBase contains common dependencies for CRI's runtime, image, and podsandbox services. type CRIBase struct { // Config contains all configurations. Config criconfig.Config - // Client is an instance of the containerd client - Client *containerd.Client + // BaseOCISpecs contains cached OCI specs loaded via `Runtime.BaseRuntimeSpec` + BaseOCISpecs map[string]*oci.Spec } func init() { @@ -47,13 +52,23 @@ func init() { // Base plugin that other CRI services depend on. registry.Register(&plugin.Registration{ - Type: plugins.GRPCPlugin, + Type: plugins.InternalPlugin, ID: "cri", Config: &config, Requires: []plugin.Type{ - plugins.EventPlugin, - plugins.ServicePlugin, - plugins.NRIApiPlugin, + plugins.WarningPlugin, + }, + ConfigMigration: func(ctx context.Context, version int, plugins map[string]interface{}) error { + if version >= srvconfig.CurrentConfigVersion { + return nil + } + c, ok := plugins["io.containerd.grpc.v1.cri"] + if !ok { + return nil + } + conf := c.(map[string]interface{}) + plugins["io.containerd.internal.v1.cri"] = conf + return nil }, InitFn: initCRIBase, }) @@ -64,8 +79,17 @@ func initCRIBase(ic *plugin.InitContext) (interface{}, error) { ic.Meta.Exports = map[string]string{"CRIVersion": constants.CRIVersion} ctx := ic.Context pluginConfig := ic.Config.(*criconfig.PluginConfig) - if err := criconfig.ValidatePluginConfig(ctx, pluginConfig); err != nil { + if warnings, err := criconfig.ValidatePluginConfig(ctx, pluginConfig); err != nil { return nil, fmt.Errorf("invalid plugin config: %w", err) + } else if len(warnings) > 0 { + ws, err := ic.GetSingle(plugins.WarningPlugin) + if err != nil { + return nil, err + } + warn := ws.(warning.Service) + for _, w := range warnings { + warn.Emit(ctx, w) + } } c := criconfig.Config{ @@ -82,23 +106,55 @@ func initCRIBase(ic *plugin.InitContext) (interface{}, error) { return nil, fmt.Errorf("failed to set glog level: %w", err) } - log.G(ctx).Info("Connect containerd service") - client, err := containerd.New( - "", - containerd.WithDefaultNamespace(constants.K8sContainerdNamespace), - containerd.WithDefaultPlatform(platforms.Default()), - containerd.WithInMemoryServices(ic), - ) + ociSpec, err := loadBaseOCISpecs(&c) if err != nil { - return nil, fmt.Errorf("failed to create containerd client: %w", err) + return nil, fmt.Errorf("failed to create load basic oci spec: %w", err) } return &CRIBase{ - Config: c, - Client: client, + Config: c, + BaseOCISpecs: ociSpec, }, nil } +func loadBaseOCISpecs(config *criconfig.Config) (map[string]*oci.Spec, error) { + specs := map[string]*oci.Spec{} + for _, cfg := range config.Runtimes { + if cfg.BaseRuntimeSpec == "" { + continue + } + + // Don't load same file twice + if _, ok := specs[cfg.BaseRuntimeSpec]; ok { + continue + } + + spec, err := loadOCISpec(cfg.BaseRuntimeSpec) + if err != nil { + return nil, fmt.Errorf("failed to load base OCI spec from file: %s: %w", cfg.BaseRuntimeSpec, err) + } + + specs[cfg.BaseRuntimeSpec] = spec + } + + return specs, nil +} + +func loadOCISpec(filename string) (*oci.Spec, error) { + file, err := os.Open(filename) + if err != nil { + return nil, fmt.Errorf("failed to open base OCI spec: %s: %w", filename, err) + } + defer file.Close() + + spec := oci.Spec{} + if err := json.NewDecoder(file).Decode(&spec); err != nil { + return nil, fmt.Errorf("failed to parse base OCI spec file: %w", err) + } + + return &spec, nil +} + // Set glog level. func setGLogLevel() error { l := log.GetLevel() diff --git a/pkg/cri/server/base/cri_base_test.go b/pkg/cri/server/base/cri_base_test.go new file mode 100644 index 000000000000..a30c1bbb44e1 --- /dev/null +++ b/pkg/cri/server/base/cri_base_test.go @@ -0,0 +1,60 @@ +/* + Copyright The containerd Authors. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +package base + +import ( + "encoding/json" + "os" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/containerd/containerd/v2/oci" + criconfig "github.com/containerd/containerd/v2/pkg/cri/config" +) + +func TestLoadBaseOCISpec(t *testing.T) { + spec := oci.Spec{Version: "1.0.2", Hostname: "default"} + + file, err := os.CreateTemp("", "spec-test-") + require.NoError(t, err) + + defer func() { + assert.NoError(t, file.Close()) + assert.NoError(t, os.RemoveAll(file.Name())) + }() + + err = json.NewEncoder(file).Encode(&spec) + assert.NoError(t, err) + + config := criconfig.Config{} + config.Runtimes = map[string]criconfig.Runtime{ + "runc": {BaseRuntimeSpec: file.Name()}, + } + + specs, err := loadBaseOCISpecs(&config) + assert.NoError(t, err) + + assert.Len(t, specs, 1) + + out, ok := specs[file.Name()] + assert.True(t, ok, "expected spec with file name %q", file.Name()) + + assert.Equal(t, "1.0.2", out.Version) + assert.Equal(t, "default", out.Hostname) +} diff --git a/pkg/cri/server/images/service.go b/pkg/cri/server/images/service.go index 7373c0de560c..f81d6128cf19 100644 --- a/pkg/cri/server/images/service.go +++ b/pkg/cri/server/images/service.go @@ -30,6 +30,7 @@ import ( containerd "github.com/containerd/containerd/v2/client" criconfig "github.com/containerd/containerd/v2/pkg/cri/config" + "github.com/containerd/containerd/v2/pkg/cri/constants" "github.com/containerd/containerd/v2/pkg/cri/server/base" imagestore "github.com/containerd/containerd/v2/pkg/cri/store/image" snapshotstore "github.com/containerd/containerd/v2/pkg/cri/store/snapshot" @@ -42,18 +43,33 @@ import ( func init() { registry.Register(&plugin.Registration{ - Type: plugins.GRPCPlugin, + Type: plugins.CRIImagePlugin, ID: "cri-image-service", + Requires: []plugin.Type{ + plugins.LeasePlugin, + plugins.EventPlugin, + plugins.SandboxStorePlugin, + plugins.InternalPlugin, + plugins.ServicePlugin, + }, InitFn: func(ic *plugin.InitContext) (interface{}, error) { // Get base CRI dependencies. - criPlugin, err := ic.GetByID(plugins.GRPCPlugin, "cri") + criPlugin, err := ic.GetByID(plugins.InternalPlugin, "cri") if err != nil { return nil, fmt.Errorf("unable to load CRI service base dependencies: %w", err) } - cri := criPlugin.(*base.CRIBase) - service, err := NewService(cri.Config, cri.Client) + client, err := containerd.New( + "", + containerd.WithDefaultNamespace(constants.K8sContainerdNamespace), + containerd.WithDefaultPlatform(platforms.Default()), + containerd.WithInMemoryServices(ic), + ) + if err != nil { + return nil, fmt.Errorf("unable to init client for cri image service: %w", err) + } + service, err := NewService(cri.Config, client) if err != nil { return nil, fmt.Errorf("failed to create image service: %w", err) } diff --git a/pkg/cri/server/podsandbox/controller.go b/pkg/cri/server/podsandbox/controller.go index 0d37179d3f55..3715ce2222cb 100644 --- a/pkg/cri/server/podsandbox/controller.go +++ b/pkg/cri/server/podsandbox/controller.go @@ -22,6 +22,8 @@ import ( "time" "github.com/containerd/log" + "github.com/containerd/plugin" + "github.com/containerd/plugin/registry" runtime "k8s.io/cri-api/pkg/apis/runtime/v1" eventtypes "github.com/containerd/containerd/v2/api/events" @@ -30,6 +32,8 @@ import ( "github.com/containerd/containerd/v2/oci" criconfig "github.com/containerd/containerd/v2/pkg/cri/config" "github.com/containerd/containerd/v2/pkg/cri/constants" + "github.com/containerd/containerd/v2/pkg/cri/server/base" + "github.com/containerd/containerd/v2/pkg/cri/server/images" imagestore "github.com/containerd/containerd/v2/pkg/cri/store/image" sandboxstore "github.com/containerd/containerd/v2/pkg/cri/store/sandbox" ctrdutil "github.com/containerd/containerd/v2/pkg/cri/util" @@ -38,8 +42,6 @@ import ( "github.com/containerd/containerd/v2/plugins" "github.com/containerd/containerd/v2/protobuf" "github.com/containerd/containerd/v2/sandbox" - "github.com/containerd/plugin" - "github.com/containerd/plugin/registry" ) func init() { @@ -49,20 +51,44 @@ func init() { Requires: []plugin.Type{ plugins.EventPlugin, plugins.LeasePlugin, + plugins.SandboxStorePlugin, + plugins.InternalPlugin, + plugins.CRIImagePlugin, plugins.ServicePlugin, }, InitFn: func(ic *plugin.InitContext) (interface{}, error) { - c := Controller{} client, err := containerd.New( "", containerd.WithDefaultNamespace(constants.K8sContainerdNamespace), containerd.WithDefaultPlatform(platforms.Default()), containerd.WithInMemoryServices(ic), ) + if err != nil { + return nil, fmt.Errorf("unable to init client for podsandbox: %w", err) + } + + // Get base CRI dependencies. + criBasePlugin, err := ic.GetByID(plugins.InternalPlugin, "cri") if err != nil { return nil, fmt.Errorf("unable to load CRI service base dependencies: %w", err) } - c.client = client + criBase := criBasePlugin.(*base.CRIBase) + + // Get image service. + criImagePlugin, err := ic.GetByID(plugins.CRIImagePlugin, "cri-image-service") + if err != nil { + return nil, fmt.Errorf("unable to load CRI image service plugin dependency: %w", err) + } + imageService := criImagePlugin.(*images.CRIImageService) + + c := Controller{ + client: client, + config: criBase.Config, + os: osinterface.RealOS{}, + baseOCISpecs: criBase.BaseOCISpecs, + imageService: imageService, + store: NewStore(), + } return &c, nil }, }) @@ -103,20 +129,11 @@ type Controller struct { } func (c *Controller) Init( - config criconfig.Config, sandboxStore *sandboxstore.Store, - os osinterface.OS, cri CRIService, - imageService ImageService, - baseOCISpecs map[string]*oci.Spec, ) { c.cri = cri - c.config = config c.sandboxStore = sandboxStore - c.os = os - c.baseOCISpecs = baseOCISpecs - c.store = NewStore() - c.imageService = imageService } var _ sandbox.Controller = (*Controller)(nil) diff --git a/pkg/cri/server/service.go b/pkg/cri/server/service.go index 8fb40e0e820c..0f32f1e50750 100644 --- a/pkg/cri/server/service.go +++ b/pkg/cri/server/service.go @@ -18,12 +18,9 @@ package server import ( "context" - "encoding/json" "fmt" "io" "net/http" - "os" - "path/filepath" "sync" "sync/atomic" @@ -38,6 +35,7 @@ import ( criconfig "github.com/containerd/containerd/v2/pkg/cri/config" "github.com/containerd/containerd/v2/pkg/cri/instrument" "github.com/containerd/containerd/v2/pkg/cri/nri" + "github.com/containerd/containerd/v2/pkg/cri/server/base" "github.com/containerd/containerd/v2/pkg/cri/server/podsandbox" containerstore "github.com/containerd/containerd/v2/pkg/cri/store/container" imagestore "github.com/containerd/containerd/v2/pkg/cri/store/image" @@ -47,7 +45,6 @@ import ( ctrdutil "github.com/containerd/containerd/v2/pkg/cri/util" osinterface "github.com/containerd/containerd/v2/pkg/os" "github.com/containerd/containerd/v2/pkg/registrar" - "github.com/containerd/containerd/v2/plugins" "github.com/containerd/containerd/v2/sandbox" ) @@ -134,15 +131,17 @@ type criService struct { } // NewCRIService returns a new instance of CRIService -func NewCRIService(config criconfig.Config, imageService imageService, client *containerd.Client, nri *nri.API) (CRIService, error) { +func NewCRIService(criBase *base.CRIBase, imageService imageService, client *containerd.Client, nri *nri.API) (CRIService, error) { var err error labels := label.NewStore() + config := criBase.Config c := &criService{ imageService: imageService, config: config, client: client, imageFSPaths: imageService.ImageFSPaths(), os: osinterface.RealOS{}, + baseOCISpecs: criBase.BaseOCISpecs, sandboxStore: sandboxstore.NewStore(labels), containerStore: containerstore.NewStore(labels), sandboxNameIndex: registrar.NewRegistrar(), @@ -183,15 +182,8 @@ func NewCRIService(config criconfig.Config, imageService imageService, client *c } } - // Preload base OCI specs - c.baseOCISpecs, err = loadBaseOCISpecs(&config) - if err != nil { - return nil, err - } - podSandboxController := client.SandboxController(string(criconfig.ModePodSandbox)).(*podsandbox.Controller) - // Initialize pod sandbox controller - podSandboxController.Init(config, c.sandboxStore, c.os, c, c.imageService, c.baseOCISpecs) + podSandboxController.Init(c.sandboxStore, c) c.nri = nri @@ -335,50 +327,3 @@ func (c *criService) register(s *grpc.Server) error { runtime.RegisterImageServiceServer(s, instrumented) return nil } - -// imageFSPath returns containerd image filesystem path. -// Note that if containerd changes directory layout, we also needs to change this. -func imageFSPath(rootDir, snapshotter string) string { - return filepath.Join(rootDir, plugins.SnapshotPlugin.String()+"."+snapshotter) -} - -func loadOCISpec(filename string) (*oci.Spec, error) { - file, err := os.Open(filename) - if err != nil { - return nil, fmt.Errorf("failed to open base OCI spec: %s: %w", filename, err) - } - defer file.Close() - - spec := oci.Spec{} - if err := json.NewDecoder(file).Decode(&spec); err != nil { - return nil, fmt.Errorf("failed to parse base OCI spec file: %w", err) - } - - return &spec, nil -} - -func loadBaseOCISpecs(config *criconfig.Config) (map[string]*oci.Spec, error) { - specs := map[string]*oci.Spec{} - for _, cfg := range config.Runtimes { - if cfg.BaseRuntimeSpec == "" { - continue - } - - // Don't load same file twice - if _, ok := specs[cfg.BaseRuntimeSpec]; ok { - continue - } - - spec, err := loadOCISpec(cfg.BaseRuntimeSpec) - if err != nil { - return nil, fmt.Errorf("failed to load base OCI spec from file: %s: %w", cfg.BaseRuntimeSpec, err) - } - - if spec.Process != nil && spec.Process.Capabilities != nil && len(spec.Process.Capabilities.Inheritable) > 0 { - log.L.WithField("base_runtime_spec", cfg.BaseRuntimeSpec).Warn("Provided base runtime spec includes inheritable capabilities, which may be unsafe. See CVE-2022-24769 for more details.") - } - specs[cfg.BaseRuntimeSpec] = spec - } - - return specs, nil -} diff --git a/pkg/cri/server/service_test.go b/pkg/cri/server/service_test.go index 0800dc73eec7..b231f511d181 100644 --- a/pkg/cri/server/service_test.go +++ b/pkg/cri/server/service_test.go @@ -17,25 +17,13 @@ package server import ( - "bytes" "context" - "encoding/json" - "io" - "os" - "testing" "github.com/containerd/go-cni" - "github.com/containerd/log" - "github.com/opencontainers/runtime-spec/specs-go" - "github.com/sirupsen/logrus" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" runtime "k8s.io/cri-api/pkg/apis/runtime/v1" "github.com/containerd/containerd/v2/api/types" "github.com/containerd/containerd/v2/errdefs" - "github.com/containerd/containerd/v2/oci" - criconfig "github.com/containerd/containerd/v2/pkg/cri/config" containerstore "github.com/containerd/containerd/v2/pkg/cri/store/container" "github.com/containerd/containerd/v2/pkg/cri/store/label" sandboxstore "github.com/containerd/containerd/v2/pkg/cri/store/sandbox" @@ -103,88 +91,3 @@ func newTestCRIService() *criService { sandboxService: &fakeSandboxService{}, } } - -func TestLoadBaseOCISpec(t *testing.T) { - spec := oci.Spec{Version: "1.0.2", Hostname: "default"} - - file, err := os.CreateTemp("", "spec-test-") - require.NoError(t, err) - - defer func() { - assert.NoError(t, file.Close()) - assert.NoError(t, os.RemoveAll(file.Name())) - }() - - err = json.NewEncoder(file).Encode(&spec) - assert.NoError(t, err) - - config := criconfig.Config{} - config.Runtimes = map[string]criconfig.Runtime{ - "runc": {BaseRuntimeSpec: file.Name()}, - } - - specs, err := loadBaseOCISpecs(&config) - assert.NoError(t, err) - - assert.Len(t, specs, 1) - - out, ok := specs[file.Name()] - assert.True(t, ok, "expected spec with file name %q", file.Name()) - - assert.Equal(t, "1.0.2", out.Version) - assert.Equal(t, "default", out.Hostname) -} - -func Test_loadBaseOCISpecs(t *testing.T) { - spec := oci.Spec{ - Version: "1.0.2", - Hostname: "default", - Process: &specs.Process{ - Capabilities: &specs.LinuxCapabilities{ - Inheritable: []string{"CAP_NET_RAW"}, - }, - }, - } - file, err := os.CreateTemp("", "spec-test-") - require.NoError(t, err) - defer func() { - assert.NoError(t, file.Close()) - assert.NoError(t, os.RemoveAll(file.Name())) - }() - err = json.NewEncoder(file).Encode(&spec) - require.NoError(t, err) - config := criconfig.Config{} - config.Runtimes = map[string]criconfig.Runtime{ - "runc": {BaseRuntimeSpec: file.Name()}, - } - var buffer bytes.Buffer - logger := &logrus.Logger{ - Out: &buffer, - Formatter: new(logrus.TextFormatter), - Hooks: make(logrus.LevelHooks), - Level: logrus.InfoLevel, - ExitFunc: os.Exit, - ReportCaller: false, - } - log.L = logrus.NewEntry(logger) - tests := []struct { - name string - args *criconfig.Config - message string - }{ - { - name: "args is not nil,print warning", - args: &config, - message: "Provided base runtime spec includes inheritable capabilities, which may be unsafe. See CVE-2022-24769 for more details.", - }, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - loadBaseOCISpecs(tt.args) - readAll, _ := io.ReadAll(&buffer) - if tt.message != "" { - assert.Contains(t, string(readAll), tt.message) - } - }) - } -} diff --git a/plugins/types.go b/plugins/types.go index 1ac4a27375e9..dfe9b9666dd5 100644 --- a/plugins/types.go +++ b/plugins/types.go @@ -67,6 +67,8 @@ const ( ImageVerifierPlugin plugin.Type = "io.containerd.image-verifier.v1" // WarningPlugin implements a warning service WarningPlugin plugin.Type = "io.containerd.warning.v1" + // CRIImagePlugin implements a cri image service + CRIImagePlugin plugin.Type = "io.containerd.cri.image.v1" ) const ( From b0fef6738f728ff2b613e81c04d9a0928bf37cbf Mon Sep 17 00:00:00 2001 From: Abel Feng Date: Fri, 20 Oct 2023 22:01:21 +0800 Subject: [PATCH 3/4] sandbox: migrate sandbox_mode to sandboxer Signed-off-by: Abel Feng --- pkg/cri/server/base/cri_base.go | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/pkg/cri/server/base/cri_base.go b/pkg/cri/server/base/cri_base.go index 553393774030..dc897e071fbc 100644 --- a/pkg/cri/server/base/cri_base.go +++ b/pkg/cri/server/base/cri_base.go @@ -67,6 +67,7 @@ func init() { return nil } conf := c.(map[string]interface{}) + migrateConfig(conf) plugins["io.containerd.internal.v1.cri"] = conf return nil }, @@ -175,3 +176,22 @@ func setGLogLevel() error { } return nil } + +func migrateConfig(conf map[string]interface{}) { + containerdConf, ok := conf["containerd"] + if !ok { + return + } + runtimesConf, ok := containerdConf.(map[string]interface{})["runtimes"] + if !ok { + return + } + for _, v := range runtimesConf.(map[string]interface{}) { + runtimeConf := v.(map[string]interface{}) + if sandboxMode, ok := runtimeConf["sandbox_mode"]; ok { + if _, ok := runtimeConf["sandboxer"]; !ok { + runtimeConf["sandboxer"] = sandboxMode + } + } + } +} From e1b49586633c4ad2a78ff91abc10a92d7bbdae7d Mon Sep 17 00:00:00 2001 From: Abel Feng Date: Mon, 20 Nov 2023 22:24:38 +0800 Subject: [PATCH 4/4] sandbox: keep rootDir and stateDir compatible Signed-off-by: Abel Feng --- pkg/cri/server/base/cri_base.go | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/pkg/cri/server/base/cri_base.go b/pkg/cri/server/base/cri_base.go index dc897e071fbc..09e9de8995d2 100644 --- a/pkg/cri/server/base/cri_base.go +++ b/pkg/cri/server/base/cri_base.go @@ -93,12 +93,17 @@ func initCRIBase(ic *plugin.InitContext) (interface{}, error) { } } + // For backward compatibility, we have to keep the rootDir and stateDir the same as before. + containerdRootDir := filepath.Dir(ic.Properties[plugins.PropertyRootDir]) + rootDir := filepath.Join(containerdRootDir, "io.containerd.grpc.v1.cri") + containerdStateDir := filepath.Dir(ic.Properties[plugins.PropertyStateDir]) + stateDir := filepath.Join(containerdStateDir, "io.containerd.grpc.v1.cri") c := criconfig.Config{ PluginConfig: *pluginConfig, - ContainerdRootDir: filepath.Dir(ic.Properties[plugins.PropertyRootDir]), + ContainerdRootDir: containerdRootDir, ContainerdEndpoint: ic.Properties[plugins.PropertyGRPCAddress], - RootDir: ic.Properties[plugins.PropertyRootDir], - StateDir: ic.Properties[plugins.PropertyStateDir], + RootDir: rootDir, + StateDir: stateDir, } log.G(ctx).Infof("Start cri plugin with config %+v", c)