Add RuntimeHandler support

cri.go

@@ -71,6 +71,10 @@ func initCRIService(ic *plugin.InitContext) (interface{}, error) {
 	}
 	log.G(ctx).Infof("Start cri plugin with config %+v", c)
 
+	if err := validateConfig(&c); err != nil {
+		return nil, errors.Wrap(err, "invalid config")
+	}
+
 	if err := setGLogLevel(); err != nil {
 		return nil, errors.Wrap(err, "failed to set glog level")
 	}
@@ -104,6 +108,18 @@ func initCRIService(ic *plugin.InitContext) (interface{}, error) {
 	return s, nil
 }
 
+// validateConfig validates the given configuration.
+func validateConfig(c *criconfig.Config) error {
+	// It is an error to provide both an UntrustedWorkloadRuntime & define an 'untrusted' runtime.
+	if _, ok := c.ContainerdConfig.Runtimes[criconfig.RuntimeUntrusted]; ok {
+		if c.ContainerdConfig.UntrustedWorkloadRuntime.Type != "" {
+			return errors.New("conflicting definitions: configuration includes untrusted_workload_runtime and runtimes['untrusted']")
+		}
+	}
+
+	return nil
+}
+
 // getServicesOpts get service options from plugin context.
 func getServicesOpts(ic *plugin.InitContext) ([]containerd.ServicesOpt, error) {
 	plugins, err := ic.GetByType(plugin.ServicePlugin)

docs/config.md

@@ -61,6 +61,11 @@ The explanation and default value of each configuration item are as follows:
       runtime_root = ""
 
     # "plugins.cri.containerd.untrusted_workload_runtime" is a runtime to run untrusted workloads on it.
+
+    # DEPRECATED: use plugins.cri.runtimes instead. If provided, this runtime is mapped to the
+    #     runtime handler named 'untrusted'. It is a configuration error to provide both the (now
+    #     deprecated) UntrustedWorkloadRuntime and a handler in the Runtimes handler map (below) for
+    #     'untrusted' workloads at the same time. Please provide one or the other.
     [plugins.cri.containerd.untrusted_workload_runtime]
       # runtime_type is the runtime type to use in containerd e.g. io.containerd.runtime.v1.linux
       runtime_type = ""
@@ -71,6 +76,19 @@ The explanation and default value of each configuration item are as follows:
       # runtime_root is the directory used by containerd for runtime state.
       runtime_root = ""
 
+  	# plugins.cri.containerd.runtimes is a map from CRI RuntimeHandler strings, which specify types
+  	# of runtime configurations, to the matching configurations. In this example,
+  	# 'runtime_handler_name' is the RuntimeHandler string to match.
+    [plugins.cri.containerd.runtimes.runtime_handler_name]
+      # runtime_type is the runtime type to use in containerd e.g. io.containerd.runtime.v1.linux
+      runtime_type = ""
+
+      # runtime_engine is the name of the runtime engine used by containerd.
+      runtime_engine = ""
+
+      # runtime_root is the directory used by containerd for runtime state.
+      runtime_root = ""
+
   # "plugins.cri.cni" contains config related to cni
   [plugins.cri.cni]
     # bin_dir is the directory in which the binaries for the plugin is kept.

pkg/config/config.go

@@ -33,10 +33,18 @@ type Runtime struct {
 type ContainerdConfig struct {
 	// Snapshotter is the snapshotter used by containerd.
 	Snapshotter string `toml:"snapshotter" json:"snapshotter"`
-	// DefaultRuntime is the runtime to use in containerd.
+	// DefaultRuntime is the default runtime to use in containerd.
+	// This runtime is used when no runtime handler (or the empty string) is provided.
 	DefaultRuntime Runtime `toml:"default_runtime" json:"defaultRuntime"`
 	// UntrustedWorkloadRuntime is a runtime to run untrusted workloads on it.
+	// DEPRECATED: use Runtimes instead. If provided, this runtime is mapped to the runtime handler
+	//     named 'untrusted'. It is a configuration error to provide both the (now deprecated)
+	//     UntrustedWorkloadRuntime and a handler in the Runtimes handler map (below) for 'untrusted'
+	//     workloads at the same time. Please provide one or the other.
 	UntrustedWorkloadRuntime Runtime `toml:"untrusted_workload_runtime" json:"untrustedWorkloadRuntime"`
+	// Runtimes is a map from CRI RuntimeHandler strings, which specify types of runtime
+	// configurations, to the matching configurations.
+	Runtimes map[string]Runtime `toml:"runtimes" json:"runtimes"`
 	// NoPivot disables pivot-root (linux only), required when running a container in a RamDisk with runc
 	NoPivot bool `toml:"no_pivot" json:"noPivot"`
 }
@@ -183,3 +191,8 @@ func DefaultConfig() PluginConfig {
 		},
 	}
 }
+
+const (
+	// RuntimeUntrusted is the implicit runtime defined for ContainerdConfig.UntrustedWorkloadRuntime
+	RuntimeUntrusted = "untrusted"
+)

pkg/server/sandbox_run.go

@@ -131,7 +131,7 @@ func (c *criService) RunPodSandbox(ctx context.Context, r *runtime.RunPodSandbox
 		}()
 	}
 
-	ociRuntime, err := c.getSandboxRuntime(config)
+	ociRuntime, err := c.getSandboxRuntime(config, r.GetRuntimeHandler())
 	if err != nil {
 		return nil, errors.Wrap(err, "failed to get sandbox runtime")
 	}
@@ -601,9 +601,13 @@ func hostAccessingSandbox(config *runtime.PodSandboxConfig) bool {
 // getSandboxRuntime returns the runtime configuration for sandbox.
 // If the sandbox contains untrusted workload, runtime for untrusted workload will be returned,
 // or else default runtime will be returned.
-func (c *criService) getSandboxRuntime(config *runtime.PodSandboxConfig) (criconfig.Runtime, error) {
-	untrusted := false
+func (c *criService) getSandboxRuntime(config *runtime.PodSandboxConfig, runtimeHandler string) (criconfig.Runtime, error) {
 	if untrustedWorkload(config) {
+		// If the untrusted annotation is provided, runtimeHandler MUST be empty.
+		if runtimeHandler != "" && runtimeHandler != criconfig.RuntimeUntrusted {
+			return criconfig.Runtime{}, errors.New("untrusted workload with explicit runtime handler is not allowed")
+		}
+
 		//  If the untrusted workload is requesting access to the host/node, this request will fail.
 		//
 		//  Note: If the workload is marked untrusted but requests privileged, this can be granted, as the
@@ -612,14 +616,22 @@ func (c *criService) getSandboxRuntime(config *runtime.PodSandboxConfig) (cricon
 		if hostAccessingSandbox(config) {
 			return criconfig.Runtime{}, errors.New("untrusted workload with host access is not allowed")
 		}
-		untrusted = true
-	}
 
-	if untrusted {
-		if c.config.ContainerdConfig.UntrustedWorkloadRuntime.Type == "" {
-			return criconfig.Runtime{}, errors.New("no runtime for untrusted workload is configured")
+		// Handle the deprecated UntrustedWorkloadRuntime.
+		if c.config.ContainerdConfig.UntrustedWorkloadRuntime.Type != "" {
+			return c.config.ContainerdConfig.UntrustedWorkloadRuntime, nil
 		}
-		return c.config.ContainerdConfig.UntrustedWorkloadRuntime, nil
+
+		runtimeHandler = criconfig.RuntimeUntrusted
+	}
+
+	if runtimeHandler == "" {
+		return c.config.ContainerdConfig.DefaultRuntime, nil
+	}
+
+	handler, ok := c.config.ContainerdConfig.Runtimes[runtimeHandler]
+	if !ok {
+		return criconfig.Runtime{}, errors.Errorf("no runtime for %q is configured", runtimeHandler)
 	}
-	return c.config.ContainerdConfig.DefaultRuntime, nil
+	return handler, nil
 }

pkg/server/sandbox_run_test.go

@@ -533,10 +533,18 @@ func TestGetSandboxRuntime(t *testing.T) {
 		Root:   "",
 	}
 
+	fooRuntime := criconfig.Runtime{
+		Type:   "io.containerd.runtime.v1.linux",
+		Engine: "foo-bar",
+		Root:   "",
+	}
+
 	for desc, test := range map[string]struct {
 		sandboxConfig            *runtime.PodSandboxConfig
+		runtimeHandler           string
 		defaultRuntime           criconfig.Runtime
 		untrustedWorkloadRuntime criconfig.Runtime
+		runtimes                 map[string]criconfig.Runtime
 		expectErr                bool
 		expectedRuntime          criconfig.Runtime
 	}{
@@ -595,6 +603,54 @@ func TestGetSandboxRuntime(t *testing.T) {
 			defaultRuntime: defaultRuntime,
 			expectErr:      true,
 		},
+		"should use 'untrusted' runtime for untrusted workload": {
+			sandboxConfig: &runtime.PodSandboxConfig{
+				Annotations: map[string]string{
+					annotations.UntrustedWorkload: "true",
+				},
+			},
+			defaultRuntime:  defaultRuntime,
+			runtimes:        map[string]criconfig.Runtime{criconfig.RuntimeUntrusted: untrustedWorkloadRuntime},
+			expectedRuntime: untrustedWorkloadRuntime,
+		},
+		"should use 'untrusted' runtime for untrusted workload & handler": {
+			sandboxConfig: &runtime.PodSandboxConfig{
+				Annotations: map[string]string{
+					annotations.UntrustedWorkload: "true",
+				},
+			},
+			runtimeHandler:  "untrusted",
+			defaultRuntime:  defaultRuntime,
+			runtimes:        map[string]criconfig.Runtime{criconfig.RuntimeUntrusted: untrustedWorkloadRuntime},
+			expectedRuntime: untrustedWorkloadRuntime,
+		},
+		"should return an error if untrusted annotation with conflicting handler": {
+			sandboxConfig: &runtime.PodSandboxConfig{
+				Annotations: map[string]string{
+					annotations.UntrustedWorkload: "true",
+				},
+			},
+			runtimeHandler:           "foo",
+			defaultRuntime:           defaultRuntime,
+			untrustedWorkloadRuntime: untrustedWorkloadRuntime,
+			runtimes:                 map[string]criconfig.Runtime{"foo": fooRuntime},
+			expectErr:                true,
+		},
+		"should use correct runtime for a runtime handler": {
+			sandboxConfig:            &runtime.PodSandboxConfig{},
+			runtimeHandler:           "foo",
+			defaultRuntime:           defaultRuntime,
+			untrustedWorkloadRuntime: untrustedWorkloadRuntime,
+			runtimes:                 map[string]criconfig.Runtime{"foo": fooRuntime},
+			expectedRuntime:          fooRuntime,
+		},
+		"should return error if runtime handler is required but not configured": {
+			sandboxConfig:  &runtime.PodSandboxConfig{},
+			runtimeHandler: "bar",
+			defaultRuntime: defaultRuntime,
+			runtimes:       map[string]criconfig.Runtime{"foo": fooRuntime},
+			expectErr:      true,
+		},
 	} {
 		t.Run(desc, func(t *testing.T) {
 			cri := newTestCRIService()
@@ -603,7 +659,8 @@ func TestGetSandboxRuntime(t *testing.T) {
 			}
 			cri.config.ContainerdConfig.DefaultRuntime = test.defaultRuntime
 			cri.config.ContainerdConfig.UntrustedWorkloadRuntime = test.untrustedWorkloadRuntime
-			r, err := cri.getSandboxRuntime(test.sandboxConfig)
+			cri.config.ContainerdConfig.Runtimes = test.runtimes
+			r, err := cri.getSandboxRuntime(test.sandboxConfig, test.runtimeHandler)
 			assert.Equal(t, test.expectErr, err != nil)
 			assert.Equal(t, test.expectedRuntime, r)
 		})

vendor.conf

@@ -69,9 +69,9 @@ google.golang.org/genproto d80a6e20e776b0b17a324d0ba1ab50a39c8e8944
 google.golang.org/grpc v1.12.0
 gopkg.in/inf.v0 3887ee99ecf07df5b447e9b00d9c0b2adaa9f3e4
 gopkg.in/yaml.v2 53feefa2559fb8dfa8d81baad31be332c97d6c77
-k8s.io/api 9e5ffd1f1320950b238cfce291b926411f0af722
-k8s.io/apimachinery ed135c5b96450fd24e5e981c708114fbbd950697
-k8s.io/apiserver a90e3a95c2e91b944bfca8225c4e0d12e42a9eb5
-k8s.io/client-go 03bfb9bdcfe5482795b999f39ca3ed9ad42ce5bb
-k8s.io/kubernetes v1.11.0
-k8s.io/utils 733eca437aa39379e4bcc25e726439dfca40fcff
+k8s.io/api 12ee108019c2efdc50febb4937fe054aede5d660
+k8s.io/apimachinery c6b66c9c507abbefa93ad83f7fe8c9b52ca1ae30
+k8s.io/apiserver ecbc9eada272b5735d74d2dd0f944dfdefdbb2a5
+k8s.io/client-go 22e1ddcc4852ed93b2c34ef13fbb287f794200ae
+k8s.io/kubernetes 6b7c39a4f8d4c38e8724550cc3e6e41b7ac7a276
+k8s.io/utils 982821ea41da7e7c15f3d3738921eb2e7e241ccd