feature: support cgroup systemd driver

support systemd cgroup driver for pouch container.

Signed-off-by: Ace-Tang <[email protected]>

ctrd/container.go

@@ -537,8 +537,9 @@ func (c *Client) createContainer(ctx context.Context, ref, id, checkpointDir str
 		containerd.WithSnapshotter(CurrentSnapshotterName(ctx)),
 		containerd.WithContainerLabels(container.Labels),
 		containerd.WithRuntime(fmt.Sprintf("io.containerd.runtime.v1.%s", runtime.GOOS), &runctypes.RuncOptions{
-			Runtime:     container.Runtime,
-			RuntimeRoot: runtimeRoot,
+			Runtime:       container.Runtime,
+			RuntimeRoot:   runtimeRoot,
+			SystemdCgroup: container.UseSystemd,
 		}),
 	}
 

ctrd/container_types.go

@@ -24,6 +24,9 @@ type Container struct {
 
 	// RootFSProvided is a flag to point the container is created by specifying rootfs
 	RootFSProvided bool
+
+	// UseSystemd tells whether container use systemd cgroup driver
+	UseSystemd bool
 }
 
 // Process wraps exec process's info.

daemon/config/config.go

@@ -22,8 +22,12 @@ import (
 )
 
 const (
-	// CgroupfsDriverType refers to daemon's cgroup driver.
-	CgroupfsDriverType = "cgroupfs"
+	// CgroupfsDriver is cgroupfs driver
+	CgroupfsDriver = "cgroupfs"
+	// CgroupSystemdDriver is systemd driver
+	CgroupSystemdDriver = "systemd"
+	// DefaultCgroupDriver is default cgroups driver
+	DefaultCgroupDriver = CgroupfsDriver
 )
 
 // Config refers to daemon's whole configurations.
@@ -120,14 +124,19 @@ type Config struct {
 
 	// AllowMultiSnapshotter allows multi snapshotter, default false
 	AllowMultiSnapshotter bool `json:"allow-multi-snapshotter,omitempty"`
+
+	// CgroupDriver sets cgroup driver for all containers
+	CgroupDriver string `json:"cgroup-driver,omitempty"`
 }
 
 // GetCgroupDriver gets cgroup driver used in runc.
 func (cfg *Config) GetCgroupDriver() string {
-	// current pouchd only supports directly managing cgroupfs.
-	// TODO: add 'systemd' to make systemd manage cgroupfs rather than directly using it.
-	// In the future we will support this config in the daemon configuration.
-	return CgroupfsDriverType
+	return cfg.CgroupDriver
+}
+
+// UseSystemd tells whether use systemd cgroup driver
+func (cfg *Config) UseSystemd() bool {
+	return cfg.CgroupDriver == CgroupSystemdDriver
 }
 
 // Validate validates the user input config.
@@ -161,6 +170,16 @@ func (cfg *Config) Validate() error {
 		cfg.Runtimes[cfg.DefaultRuntime] = types.Runtime{Path: cfg.DefaultRuntime}
 	}
 
+	// if cgroup driver is empty, use default cgroup driver
+	if cfg.CgroupDriver == "" {
+		cfg.CgroupDriver = DefaultCgroupDriver
+	}
+
+	// validates cgroup driver
+	if err := validateCgroupDriver(cfg.CgroupDriver); err != nil {
+		return err
+	}
+
 	return nil
 }
 
@@ -275,3 +294,12 @@ func getConflictConfigurations(flagSet *pflag.FlagSet, fileFlags map[string]inte
 func mergeConfigurations(src *Config, dest *Config) error {
 	return utils.Merge(src, dest)
 }
+
+// validateCgroupDriver validates cgroup driver
+func validateCgroupDriver(driver string) error {
+	if driver == CgroupfsDriver || driver == CgroupSystemdDriver {
+		return nil
+	}
+
+	return fmt.Errorf("invalid cgroup driver: %s, valid driver is cgroupfs or systemd", driver)
+}

daemon/config/config_test.go

@@ -302,3 +302,28 @@ func TestMergeConfigurations(t *testing.T) {
 
 	defer os.Remove(configFile)
 }
+
+func TestValidateCgroupDriver(t *testing.T) {
+	for _, tc := range []struct {
+		driver    string
+		expectErr bool
+	}{
+		{
+			driver:    CgroupfsDriver,
+			expectErr: false,
+		},
+		{
+			driver:    CgroupSystemdDriver,
+			expectErr: false,
+		},
+		{
+			driver:    "foo",
+			expectErr: true,
+		},
+	} {
+		err := validateCgroupDriver(tc.driver)
+		if tc.expectErr != (err != nil) {
+			t.Fatalf("expectd error: %v, but get %s", tc.expectErr, err)
+		}
+	}
+}

daemon/mgr/container.go

@@ -726,11 +726,12 @@ func (mgr *ContainerManager) createContainerdContainer(ctx context.Context, c *C
 	}
 
 	sw := &SpecWrapper{
-		ctrMgr:  mgr,
-		volMgr:  mgr.VolumeMgr,
-		netMgr:  mgr.NetworkMgr,
-		prioArr: prioArr,
-		argsArr: argsArr,
+		ctrMgr:     mgr,
+		volMgr:     mgr.VolumeMgr,
+		netMgr:     mgr.NetworkMgr,
+		prioArr:    prioArr,
+		argsArr:    argsArr,
+		useSystemd: mgr.Config.UseSystemd(),
 	}
 
 	if err = createSpec(ctx, c, sw); err != nil {
@@ -761,6 +762,7 @@ func (mgr *ContainerManager) createContainerdContainer(ctx context.Context, c *C
 		RootFSProvided: c.RootFSProvided,
 		BaseFS:         c.BaseFS,
 		SnapshotID:     c.SnapshotID,
+		UseSystemd:     mgr.Config.UseSystemd(),
 	}
 	c.Unlock()
 

daemon/mgr/spec.go

@@ -12,11 +12,12 @@ import (
 type SpecWrapper struct {
 	s *specs.Spec
 
-	ctrMgr  ContainerMgr
-	volMgr  VolumeMgr
-	netMgr  NetworkMgr
-	prioArr []int
-	argsArr [][]string
+	ctrMgr     ContainerMgr
+	volMgr     VolumeMgr
+	netMgr     NetworkMgr
+	prioArr    []int
+	argsArr    [][]string
+	useSystemd bool
 }
 
 // All the functions related to the spec is lock-free for container instance,

daemon/mgr/spec_linux.go

@@ -29,6 +29,9 @@ const (
 	ProfilePouchDefault = "pouch/default"
 	// ProfileNameUnconfined is a string indicating one should run a pod/containerd without a security profile.
 	ProfileNameUnconfined = "unconfined"
+
+	// defaultCgroupParent is default cgroup parent.
+	defaultCgroupParent = "pouch"
 )
 
 // Setup linux-platform-sepecific specification.
@@ -39,18 +42,21 @@ func populatePlatform(ctx context.Context, c *Container, specWrapper *SpecWrappe
 	}
 
 	// same with containerd use. or make it a variable
-	cgroupsParent := "default"
+	// set default cgroup parent
+	cgroupsParent := "/default"
+	if specWrapper.useSystemd {
+		cgroupsParent = "system.slice"
+	}
+
 	if c.HostConfig.CgroupParent != "" {
-		cgroupsParent = c.HostConfig.CgroupParent
+		cgroupsParent = filepath.Clean(c.HostConfig.CgroupParent)
 	}
 
-	// cgroupsPath must be absolute path
-	// call filepath.Clean is to avoid bad
-	// path just like../../../.../../BadPath
-	if !filepath.IsAbs(cgroupsParent) {
-		cgroupsParent = filepath.Clean("/" + cgroupsParent)
+	if specWrapper.useSystemd {
+		s.Linux.CgroupsPath = cgroupsParent + ":" + defaultCgroupParent + ":" + c.ID
+	} else {
+		s.Linux.CgroupsPath = filepath.Join(cgroupsParent, c.ID)
 	}
-	s.Linux.CgroupsPath = filepath.Join(cgroupsParent, c.ID)
 
 	s.Linux.Sysctl = c.HostConfig.Sysctls
 

main.go

@@ -125,7 +125,7 @@ func setupFlags(cmd *cobra.Command) {
 	flagSet.StringArrayVar(&logOpts, "log-opt", nil, "Set default log driver options")
 
 	// cgroup-path flag is to set parent cgroup for all containers, default is "default" staying with containerd's configuration.
-	flagSet.StringVar(&cfg.CgroupParent, "cgroup-parent", "default", "Set parent cgroup for all containers")
+	flagSet.StringVar(&cfg.CgroupParent, "cgroup-parent", "", "Set parent cgroup for all containers")
 	flagSet.StringSliceVar(&cfg.Labels, "label", []string{}, "Set metadata for Pouch daemon")
 	flagSet.BoolVar(&cfg.EnableProfiler, "enable-profiler", false, "Set if pouchd setup profiler")
 	flagSet.StringVar(&cfg.Pidfile, "pidfile", "/var/run/pouch.pid", "Save daemon pid")
@@ -136,6 +136,7 @@ func setupFlags(cmd *cobra.Command) {
 	// value is 'default'. So if IsCriEnabled is true for k8s, we should set the DefaultNamespace
 	// to k8s.io
 	flagSet.StringVar(&cfg.DefaultNamespace, "default-namespace", namespaces.Default, "default-namespace is passed to containerd, the default value is 'default'")
+	flagSet.StringVar(&cfg.CgroupDriver, "cgroup-driver", "cgroupfs", "Set cgroup driver for all containers(cgroupfs|systemd), default cgroupfs")
 }
 
 // runDaemon prepares configs, setups essential details and runs pouchd daemon.

test/environment/env.go

@@ -224,3 +224,12 @@ func IsCRIUExist() bool {
 	_, err := exec.LookPath("criu")
 	return err == nil
 }
+
+// SupportSystemdCgroupDriver checks if systemd cgroup driver is available on machine.
+func SupportSystemdCgroupDriver() bool {
+	fi, err := os.Lstat("/run/systemd/system")
+	if err != nil {
+		return false
+	}
+	return fi.IsDir()
+}

test/z_cli_daemon_test.go

@@ -3,6 +3,7 @@ package main
 import (
 	"encoding/json"
 	"fmt"
+	"io/ioutil"
 	"os"
 	"os/exec"
 	"path/filepath"
@@ -648,3 +649,30 @@ func (suite *PouchDaemonSuite) TestRecoverContainerWhenHostDown(c *check.C) {
 		c.Fatalf("failed to wait container running")
 	}
 }
+
+// TestDaemonWithSysyemdCgroupDriver tests start daemon with systemd cgroup driver
+func (suite *PouchDaemonSuite) TestDaemonWithSystemdCgroupDriver(c *check.C) {
+	SkipIfFalse(c, environment.SupportSystemdCgroupDriver)
+	tmpDir, err := ioutil.TempDir("", "cgroup-driver")
+	path := filepath.Join(tmpDir, "config.json")
+	c.Assert(err, check.IsNil)
+	cfg := struct {
+		CgroupDriver string `json:"cgroup-driver,omitempty"`
+	}{
+		CgroupDriver: "systemd",
+	}
+	c.Assert(CreateConfigFile(path, cfg), check.IsNil)
+	defer os.RemoveAll(tmpDir)
+
+	dcfg, err := StartDefaultDaemon("--config-file=" + path)
+	defer dcfg.KillDaemon()
+	c.Assert(err, check.IsNil)
+
+	result := RunWithSpecifiedDaemon(dcfg, "info")
+	c.Assert(util.PartialEqual(result.Stdout(), "systemd"), check.IsNil)
+
+	cname := "TestWithSystemdCgroupDriver"
+	ret := RunWithSpecifiedDaemon(dcfg, "run", "-d", "--name", cname, busyboxImage, "top")
+	defer RunWithSpecifiedDaemon(dcfg, "rm", "-f", cname)
+	ret.Assert(c, icmd.Success)
+}